It is sometimes useful to test Patchwork on real data. However, storing emails of real people in a way that is recognized by Patchwork may cause problems (ex. accidental spam). Add an option to ingress that anonymizes headers of imported emails. Note that email body is not anonymized, ex. Signed-Off-By commit headers are left intact. Anonymization also removes almost all headers, except for a small, explicitly specified subset.
Signed-off-by: Franciszek Stachura <[email protected]> --- cmd/pw/ingress/main.go | 34 +++++++----- pkg/mail/dependency_test.go | 4 +- pkg/mail/parser.go | 104 +++++++++++++++++++++++++++++++++--- pkg/mail/parser_test.go | 90 +++++++++++++++++++++++++++++-- pkg/mail/project_test.go | 12 ++--- pkg/mail/sender_test.go | 8 +-- pkg/mail/testutil_test.go | 8 +-- 7 files changed, 220 insertions(+), 40 deletions(-) diff --git a/cmd/pw/ingress/main.go b/cmd/pw/ingress/main.go index f951b0a..16d1494 100644 --- a/cmd/pw/ingress/main.go +++ b/cmd/pw/ingress/main.go @@ -29,9 +29,10 @@ import ( ) type CLI struct { - Stdin bool `short:"i" help:"Read one email from stdin and exit."` - Mbox bool `short:"m" help:"Read all emails in mbox format from stdin."` - ListID string `short:"l" help:"Force List-ID value instead of reading it from email headers."` + Stdin bool `short:"i" help:"Read one email from stdin and exit."` + Mbox bool `short:"m" help:"Read all emails in mbox format from stdin."` + ListID string `short:"l" help:"Force List-ID value instead of reading it from email headers."` + Anonymize bool `help:"Partially anonymize messages to provided list. ListID has to be specified."` } func (c *CLI) Run(ctx context.Context) error { @@ -50,8 +51,13 @@ func (c *CLI) Run(ctx context.Context) error { defer bus.Shutdown() ctx = db.WithBus(ctx, bus) + if c.Anonymize && c.ListID == "" { + return fmt.Errorf("anonymize requires listid") + } + if c.Stdin || c.Mbox { var dupErr *mail.DuplicateMailError + var parseErr *mail.ParseError var err error if c.Mbox { @@ -62,17 +68,17 @@ func (c *CLI) Run(ctx context.Context) error { if err != nil { break } - err = mail.ParseMail(ctx, database, msg, c.ListID) - if errors.As(err, &dupErr) { + err = mail.ParseMail(ctx, database, msg, c.Anonymize, c.ListID) + if errors.As(err, &dupErr) || errors.As(err, &parseErr) { log.Debugf("ignoring %s", err) } else if err != nil { break } } } else { - err = mail.ParseMail(ctx, database, os.Stdin, c.ListID) + err = mail.ParseMail(ctx, database, os.Stdin, c.Anonymize, c.ListID) } - if errors.As(err, &dupErr) { + if errors.As(err, &dupErr) || errors.As(err, &parseErr) { log.Debugf("ignoring %s", err) } else if err != nil && !errors.Is(err, io.EOF) { return fmt.Errorf("smtp: %w", err) @@ -80,7 +86,7 @@ func (c *CLI) Run(ctx context.Context) error { return nil } - sock, srv, err := c.startSMTPServer(ctx) + sock, srv, err := c.startSMTPServer(ctx, c.Anonymize) if err != nil { return fmt.Errorf("smtp: %w", err) } @@ -104,9 +110,10 @@ func (c *CLI) Run(ctx context.Context) error { return nil } -func (c *CLI) startSMTPServer(ctx context.Context) (net.Listener, *smtp.Server, error) { +func (c *CLI) startSMTPServer(ctx context.Context, anonymize bool) (net.Listener, *smtp.Server, error) { cfg := pw.GetConfig(ctx) - s := smtp.NewServer(&backend{ctx: ctx, listID: c.ListID}) + + s := smtp.NewServer(&backend{ctx: ctx, listID: c.ListID, anonymize: anonymize}) s.Addr = cfg.Ingress.Listen s.Domain = "localhost" s.ReadTimeout = 30 * time.Second @@ -129,8 +136,9 @@ func (c *CLI) startSMTPServer(ctx context.Context) (net.Listener, *smtp.Server, } type backend struct { - ctx context.Context - listID string + ctx context.Context + listID string + anonymize bool } func (b *backend) NewSession(c *smtp.Conn) (smtp.Session, error) { @@ -200,7 +208,7 @@ func (s *session) Data(r io.Reader) error { err = mail.ParseMail( s.backend.ctx, pw.GetDB(s.backend.ctx), - bytes.NewReader(data), s.backend.listID, + bytes.NewReader(data), s.backend.anonymize, s.backend.listID, ) if err != nil { var dupErr *mail.DuplicateMailError diff --git a/pkg/mail/dependency_test.go b/pkg/mail/dependency_test.go index f4fbec5..764325a 100644 --- a/pkg/mail/dependency_test.go +++ b/pkg/mail/dependency_test.go @@ -222,7 +222,7 @@ func TestDependencyMulti(t *testing.T) { break } buf, _ := io.ReadAll(msg) - ParseMail(ctx, database, bytes.NewReader(buf), "test.example.com") + ParseMail(ctx, database, bytes.NewReader(buf), false, "test.example.com") } var depCount int @@ -278,7 +278,7 @@ func TestDependencyMulti2(t *testing.T) { break } buf, _ := io.ReadAll(msg) - ParseMail(ctx, database, bytes.NewReader(buf), "test.example.com") + ParseMail(ctx, database, bytes.NewReader(buf), false, "test.example.com") } var depCount int diff --git a/pkg/mail/parser.go b/pkg/mail/parser.go index 6f405cd..dfbfd0c 100644 --- a/pkg/mail/parser.go +++ b/pkg/mail/parser.go @@ -7,6 +7,8 @@ package mail import ( "context" + "crypto/md5" + "encoding/base64" "fmt" "io" "strings" @@ -48,7 +50,81 @@ type parser struct { refs []string } -func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...string) error { +func anonymizeMsgIDs(msgIDs []string, toListDomain string) []string { + newMsgIDs := make([]string, 0) + for _, msgID := range msgIDs { + msgIDMd5 := md5.Sum([]byte(msgID)) + msgIDEncoded := base64.URLEncoding.EncodeToString(msgIDMd5[:]) + msgIDEncoded = fmt.Sprintf("%s@%s", msgIDEncoded, toListDomain) + newMsgIDs = append(newMsgIDs, msgIDEncoded) + } + return newMsgIDs +} + +func anonymizeAddress(address string, toListDomain string) string { + md5 := md5.Sum([]byte(address)) + encoded := base64.URLEncoding.EncodeToString(md5[:]) + return fmt.Sprintf("%s@%s", encoded, toListDomain) +} + +func anonymizeHeaders(listid string, header mail.Header, from mail.Address, msgid string) (mail.Header, *mail.Address, string) { + newHeader := mail.Header{} + + inReplyToIDs, err := header.MsgIDList("In-Reply-To") + if err != nil { + log.Warnf("failed to parse in-reply-to: %v", err) + } else if len(inReplyToIDs) != 0 { + newHeader.SetMsgIDList("In-Reply-To", anonymizeMsgIDs(inReplyToIDs, listid)) + } + + referencesIDs, err := header.MsgIDList("References") + if err != nil { + log.Warnf("failed to parse references: %v", err) + } else if len(referencesIDs) != 0 { + newHeader.SetMsgIDList("References", anonymizeMsgIDs(referencesIDs, listid)) + } + + newHeader.Set("List-ID", fmt.Sprintf("<%s>", listid)) + + from = *GetOriginalSender(&header, &from) + fromEmail := anonymizeAddress(from.Address, listid) + from = mail.Address{Name: from.Name, Address: fromEmail} + + newHeader.SetAddressList("From", []*mail.Address{&from}) + + msgid = anonymizeAddress(msgid, listid) + newHeader.SetMessageID(msgid) + + if state := header.Get("X-Patchwork-State"); state != "" { + newHeader.Set("X-Patchwork-State", state) + } + if delegate := header.Get("X-Patchwork-Delegate"); delegate != "" { + newHeader.Set("X-Patchwork-Delegate", delegate) + } + if hint := header.Get("X-Patchwork-Hint"); hint != "" { + newHeader.Set("X-Patchwork-Hint", hint) + } + + contentType, contentTypeParams, err := header.ContentType() + if err != nil { + log.Warnf("failed to parse content-type: %v", err) + } else { + newHeader.SetContentType(contentType, contentTypeParams) + } + + contentDisposition, contentDispositionParams, err := header.ContentDisposition() + if err == nil { + newHeader.SetContentDisposition(contentDisposition, contentDispositionParams) + } + + if mime := header.Get("MIME-Version"); mime != "" { + newHeader.Set("MIME-Version", mime) + } + + return newHeader, &from, msgid +} + +func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, anonymize bool, listid ...string) error { m, err := mail.CreateReader(r) if err != nil { return ParseErr("read message: %v", err) @@ -56,30 +132,42 @@ func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...str // basic sanity checks + header := m.Header + if strings.EqualFold(m.Header.Get("X-Patchwork-Hint"), "ignore") { log.Debugf("ignoring email due to hint") return nil } - subject, err := m.Header.Subject() + subject, err := header.Subject() if err != nil { return ParseErr("subject: %v", err) } - date, err := m.Header.Date() + date, err := header.Date() if err != nil { log.Warnf("date: %v", err) } if date.IsZero() { date = time.Now() } - msgid, err := m.Header.MessageID() + msgid, err := header.MessageID() if err != nil { return ParseErr("message-id: %v", err) } - from, err := mail.ParseAddress(m.Header.Get("From")) + from, err := mail.ParseAddress(header.Get("From")) if err != nil { return ParseErr("from: %v", err) } + if anonymize { + if len(listid) == 0 { + return ParseErr("anonymize but no listid") + } + + header, from, msgid = anonymizeHeaders(listid[0], header, *from, msgid) + header.SetDate(date) + header.SetSubject(subject) + } + queries, err := db.Begin(ctx, database) if err != nil { return err @@ -88,7 +176,7 @@ func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...str p := parser{ db: queries, - header: &m.Header, + header: &header, subject: subject, date: date, msgid: "<" + msgid + ">", @@ -115,7 +203,7 @@ func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...str p.parseSeriesMarker(isComment) p.version = ParseVersion(p.subject, p.prefixes) - p.refs = FindReferences(&m.Header) + p.refs = FindReferences(&header) log.Debugf("series marker: n=%d total=%d version=%d comment=%v refs=%v", p.number, p.total, p.version, isComment, p.refs) @@ -131,7 +219,7 @@ func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...str log.Debugf("no diff or comment content, skipping") return nil } - p.content.headers = FormatHeaders(&m.Header) + p.content.headers = FormatHeaders(&header) p.pullURL = ParsePullRequest(p.content.comment) switch { diff --git a/pkg/mail/parser_test.go b/pkg/mail/parser_test.go index 9d4c9c2..47836da 100644 --- a/pkg/mail/parser_test.go +++ b/pkg/mail/parser_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/getpatchwork/patchwork/pkg/db" ) var sampleDiff = "diff --git a/meep.text b/meep.text\n" + @@ -371,7 +373,7 @@ func TestAttachmentPatch(t *testing.T) { ) err := ParseMail(ctx, database, - strings.NewReader(data), "test.example.com") + strings.NewReader(data), false, "test.example.com") require.NoError(t, err) }) } @@ -415,7 +417,7 @@ func TestFindMessageID(t *testing.T) { data := "From: [email protected]\r\nSubject: test\r\n\r\nbody\r\n" database, ctx, _, _ := testDB(t, "test.example.com") _ = ParseMail(ctx, database, - strings.NewReader(data), "test.example.com") + strings.NewReader(data), false, "test.example.com") }) t.Run("header with comments", func(t *testing.T) { @@ -438,7 +440,7 @@ func TestFindMessageIDInvalidFallback(t *testing.T) { "\r\n%s", sampleDiff, ) _ = ParseMail(ctx, database, - strings.NewReader(data), "test.example.com") + strings.NewReader(data), false, "test.example.com") } func TestFindReferencesInvalidFallback(t *testing.T) { @@ -451,3 +453,85 @@ func TestFindReferencesInvalidFallback(t *testing.T) { refs := FindReferences(h) _ = refs } + +func TestAnonymize(t *testing.T) { + database, ctx, _, _ := testDB(t, "test.example.com") + + orgProj := db.Project{ + Linkname: "org-test-project", + Name: "Original Test Project", + Listid: "prod.example.com", + Listemail: "test@" + "prod.example.com", + UseTags: true, + } + err := database.NewInsert().Model(&orgProj). + Returning("*"). + Scan(context.Background()) + require.NoError(t, err) + + data := fmt.Sprintf( + "From: name <[email protected]>\r\n"+ + "Subject: [PATCH] test\r\n"+ + "Message-ID: <[email protected]>\r\n"+ + "In-Reply-To: <[email protected]>\r\n"+ + "References: <[email protected]>\r\n"+ + "List-Id: <prod.example.com>\r\n"+ + "Mime-Version: 1.0\r\n"+ + "Content-Type: text/plain\r\n"+ + "Date: Sat, 25 Oct 2025 16:08:59 +0300\r\n"+ + "DKIM-Signature: invalid\r\n"+ + "Received: invalid\r\n"+ + "X-Unknown-Header: some data\r\n"+ + "\r\n%s", sampleDiff, + ) + _ = ParseMail(ctx, database, + strings.NewReader(data), true, "test.example.com") + + var patch db.Patch + database.NewSelect().TableExpr("patch"). + Limit(1). + Scan(context.Background(), &patch) + + assert.Equal(t, "<[email protected]>", patch.Msgid) + assert.Equal(t, sampleDiff, *patch.Diff) + assert.Equal(t, "test", patch.Name) + assert.NotEqual(t, orgProj.ID, patch.ProjectID) + + for line := range strings.Lines(patch.Headers) { + split := strings.SplitN(line, ":", 2) + name := split[0] + value := strings.TrimSpace(split[1]) + + switch name { + case "From": + assert.Equal(t, "\"name\" <[email protected]>", value) + case "Message-Id": + assert.Equal(t, "<[email protected]>", patch.Msgid) + case "In-Reply-To": + assert.Equal(t, "<[email protected]>", value) + case "References": + assert.Equal(t, "<[email protected]>", value) + case "List-Id": + assert.Equal(t, "<test.example.com>", value) + case "Mime-Version": + assert.Equal(t, "1.0", value) + case "Content-Type": + assert.Equal(t, "text/plain", value) + case "Date": + assert.Equal(t, "Sat, 25 Oct 2025 16:08:59 +0300", value) + case "Subject": + assert.Equal(t, "[PATCH] test", value) + default: + assert.Fail(t, "invalid header in headers %s", name) + } + } + + var submitter db.Person + database.NewSelect().TableExpr("person"). + Limit(1). + Scan(context.Background(), &submitter) + + assert.Equal(t, submitter.ID, patch.SubmitterID) + assert.Equal(t, "name", *submitter.Name) + assert.Equal(t, "[email protected]", submitter.Email) +} diff --git a/pkg/mail/project_test.go b/pkg/mail/project_test.go index baa2044..69a9cce 100644 --- a/pkg/mail/project_test.go +++ b/pkg/mail/project_test.go @@ -62,7 +62,7 @@ func TestSubjectMatchListIDOverride(t *testing.T) { data := createEmail(sampleDiff, withSubject("[PATCH keyword] test"), withListID("nonexistent.test.org")) - err := ParseMail(ctx, database, bytes.NewReader(data), + err := ParseMail(ctx, database, bytes.NewReader(data), false, "test-subject-match.test.org") require.NoError(t, err) assert.Equal(t, 1, countPatches(t, database), "expected 1 patch with listid override") @@ -92,14 +92,14 @@ func TestListIdHeaderVariants(t *testing.T) { t.Run("blank list id", func(t *testing.T) { data := createEmail(sampleDiff, withListID("")) - err := ParseMail(ctx, database, bytes.NewReader(data)) + err := ParseMail(ctx, database, bytes.NewReader(data), false) require.NoError(t, err) assert.Equal(t, 0, countPatches(t, database), "expected 0 patches for blank list-id") }) t.Run("substring list id", func(t *testing.T) { data := createEmail(sampleDiff, withListID("example.com")) - err := ParseMail(ctx, database, bytes.NewReader(data)) + err := ParseMail(ctx, database, bytes.NewReader(data), false) require.NoError(t, err) assert.Equal(t, 0, countPatches(t, database), "expected 0 patches for substring match") }) @@ -107,7 +107,7 @@ func TestListIdHeaderVariants(t *testing.T) { t.Run("short list id", func(t *testing.T) { data := createEmail(sampleDiff) raw := strings.Replace(string(data), "List-Id: <test.example.com>", "List-Id: test.example.com", 1) - err := ParseMail(ctx, database, strings.NewReader(raw)) + err := ParseMail(ctx, database, strings.NewReader(raw), false) require.NoError(t, err) assert.Equal(t, 1, countPatches(t, database), "expected 1 patch for short list-id") }) @@ -116,7 +116,7 @@ func TestListIdHeaderVariants(t *testing.T) { data := createEmail(sampleDiff) raw := strings.Replace(string(data), "List-Id: <test.example.com>", "List-Id: Test text <test.example.com>", 1) - err := ParseMail(ctx, database, strings.NewReader(raw)) + err := ParseMail(ctx, database, strings.NewReader(raw), false) require.NoError(t, err) assert.Equal(t, 2, countPatches(t, database), "expected 2 patches for long list-id") }) @@ -128,7 +128,7 @@ func TestListIdWhitespace(t *testing.T) { data := createEmail(sampleDiff) raw := strings.Replace(string(data), "List-Id: <test.example.com>", "List-Id: ", 1) - ParseMail(ctx, database, strings.NewReader(raw)) + ParseMail(ctx, database, strings.NewReader(raw), false) assert.Equal(t, 0, countPatches(t, database), "expected 0 patches for whitespace list-id") } diff --git a/pkg/mail/sender_test.go b/pkg/mail/sender_test.go index de5e087..a2d2d21 100644 --- a/pkg/mail/sender_test.go +++ b/pkg/mail/sender_test.go @@ -81,7 +81,7 @@ func TestSenderEncodingEmpty(t *testing.T) { data := "Subject: test\r\nMessage-ID: <empty-from@test>\r\n" + "List-Id: <test.example.com>\r\n\r\n" + sampleDiff err := ParseMail(ctx, database, - strings.NewReader(data), "test.example.com") + strings.NewReader(data), false, "test.example.com") var pe *ParseError assert.ErrorAs(t, err, &pe) } @@ -148,7 +148,7 @@ func TestSenderDMARCMunging(t *testing.T) { raw := string(data) raw = strings.Replace(raw, "\r\n\r\n", "\r\nReply-To: Existing Sender <[email protected]>\r\n\r\n", 1) - ParseMail(ctx, database, strings.NewReader(raw), + ParseMail(ctx, database, strings.NewReader(raw), false, "test.example.com") var count int @@ -170,7 +170,7 @@ func TestSenderDMARCMunging(t *testing.T) { withFrom(munged), withListID("test.example.com"), withHeader("X-Original-From", "Existing Sender <[email protected]>")) - ParseMail(ctx, database, bytes.NewReader(data), + ParseMail(ctx, database, bytes.NewReader(data), false, "test.example.com") var count int @@ -194,7 +194,7 @@ func TestSenderWeirdDMARCMunging(t *testing.T) { withFrom(munged), withListID("test.example.com"), withHeader("X-Original-From", "Existing Sender <[email protected]>")) - ParseMail(ctx, database, bytes.NewReader(data), + ParseMail(ctx, database, bytes.NewReader(data), false, "test.example.com") var count int diff --git a/pkg/mail/testutil_test.go b/pkg/mail/testutil_test.go index e1b250a..4930882 100644 --- a/pkg/mail/testutil_test.go +++ b/pkg/mail/testutil_test.go @@ -156,7 +156,7 @@ func parseMbox(t *testing.T, ctx context.Context, database *bun.DB, filename str if len(listid) > 0 { lid = listid[0] } - err = ParseMail(ctx, database, bytes.NewReader(buf), lid) + err = ParseMail(ctx, database, bytes.NewReader(buf), false, lid) if err != nil { var dup *DuplicateMailError if errors.As(err, &dup) { @@ -325,7 +325,7 @@ func parseMboxTemplate(t *testing.T, ctx context.Context, database *bun.DB, file } buf, err := io.ReadAll(msg) require.NoError(t, err) - err = ParseMail(ctx, database, bytes.NewReader(buf), listid) + err = ParseMail(ctx, database, bytes.NewReader(buf), false, listid) if err != nil { var dup *DuplicateMailError if errors.As(err, &dup) { @@ -449,7 +449,7 @@ func withHeader(k, v string) emailOpt { func parseEmail(t *testing.T, ctx context.Context, database *bun.DB, body string, opts ...emailOpt) error { t.Helper() data := createEmail(body, opts...) - return ParseMail(ctx, database, bytes.NewReader(data)) + return ParseMail(ctx, database, bytes.NewReader(data), false) } // parseEml reads a single .eml or .mbox file and calls ParseMail. @@ -457,7 +457,7 @@ func parseEml(t *testing.T, ctx context.Context, database *bun.DB, filename stri t.Helper() data, err := os.ReadFile("testdata/" + filename) require.NoError(t, err) - return ParseMail(ctx, database, bytes.NewReader(data)) + return ParseMail(ctx, database, bytes.NewReader(data), false) } // countEvents drains pending events by shutting down the bus, then -- 2.55.0 _______________________________________________ Patchwork mailing list [email protected] https://lists.ozlabs.org/listinfo/patchwork
