changeset: 6909:f05df6b258f3 user: David Champion <d...@bikeshed.us> date: Mon Jan 23 19:01:36 2017 -0800 link: http://dev.mutt.org/hg/mutt/rev/f05df6b258f3
Abstract the SPAM_LIST as a generic REPLACE_LIST REPLACE_LIST can be used more generally as a list of pattern match-replace settings. SPAM_LIST was a special case of this, so spam handling has been been changed to use REPLACE_LIST instead, and SPAM_LIST was removed. A generic function for performing a REPLACE_LIST replacement has been added in mutt_apply_replace(). Commited by Kevin McCarthy with some buffer overflow fixes in mutt_apply_replace(). changeset: 6910:9e876d64d3c8 user: David Champion <d...@bikeshed.us> date: Mon Jan 23 19:01:50 2017 -0800 link: http://dev.mutt.org/hg/mutt/rev/9e876d64d3c8 Add subjectrx command to replace matching subjects with something else. This lets you define regular expressions-replacement pairs for subject display. When a Subject: matches the regular expression, the replacement value will be displayed instead in the message index. Backreferences are supported. This is especially nice for simplifying subjects that are overly wordy, such as mailing list posts (with [Listname] tags, etc), mail from ticketing systems or bug trackers, etc. It lets you reduce clutter in your mutt display without altering the messages themselves. diffs (638 lines): diff -r 7a8ea1bb09f0 -r 9e876d64d3c8 doc/manual.xml.head --- a/doc/manual.xml.head Mon Jan 23 18:46:16 2017 -0800 +++ b/doc/manual.xml.head Mon Jan 23 19:01:50 2017 -0800 @@ -6076,6 +6076,72 @@ </sect1> +<sect1 id="display-munging"> +<title>Display Munging</title> + +<para> +Working within the confines of a console or terminal window, it is +often useful to be able to modify certain information elements in a +non-destructive way -- to change how they display, without changing +the stored value of the information itself. This is especially so of +message subjects, which may often be polluted with extraneous metadata +that either is reproduced elsewhere, or is of secondary interest. +</para> + +<cmdsynopsis> +<command>subjectrx</command> +<arg choice="plain"> +<replaceable class="parameter">pattern</replaceable> +</arg> +<arg choice="plain"> +<replaceable class="parameter">replacement</replaceable> +</arg> + +<command>unsubjectrx</command> +<arg choice="plain"> +<replaceable class="parameter">pattern</replaceable> +</arg> +</cmdsynopsis> + +<para> +<literal>subjectrx</literal> specifies a regular expression +<quote>pattern</quote> which, if detected in a message subject, causes +the subject to be replaced with the <quote>replacement</quote> value. +The replacement is subject to substitutions in the same way as for the +<link linkend="spam">spam</link> command: <literal>%L</literal> for the text +to the left of the match, <literal>%R</literal> for text to the right of the +match, and <literal>%1</literal> for the first subgroup in the match (etc). +If you simply want to erase the match, set it to <quote>%L%R</quote>. +Any number of <literal>subjectrx</literal> commands may coexist. +</para> + +<para> +Note this well: the <quote>replacement</quote> value replaces the +entire subject, not just the match! +</para> + +<para> +<literal>unsubjectrx</literal> removes a given subjectrx from the substitution +list. +</para> + +<example id="ex-subjectrx"> +<title>Subject Munging</title> +<screen> +# Erase [rt #12345] tags from Request Tracker (RT) e-mails +subjectrx '\[rt #[0-9]+\] *' '%L%R' + +# Servicedesk is another RT that sends more complex subjects. +# Keep the ticket number. +subjectrx '\[servicedesk #([0-9]+)\] ([^.]+)\.([^.]+) - (new|open|pending|update) - ' '%L[#%1] %R' + +# Strip out annoying [listname] prefixes in subjects +subjectrx '\[[^\]]*\]:? *' '%L%R' +</screen> +</example> + +</sect1> + <sect1 id="new-mail"> <title>New Mail Detection</title> @@ -9991,6 +10057,23 @@ <listitem> <cmdsynopsis> +<command><link linkend="display-munging">subjectrx</link></command> +<arg choice="plain"> +<replaceable class="parameter">pattern</replaceable> +</arg> +<arg choice="plain"> +<replaceable class="parameter">format</replaceable> +</arg> + +<command><link linkend="display-munging">unsubjectrx</link></command> +<arg choice="plain"> +<replaceable class="parameter">pattern</replaceable> +</arg> +</cmdsynopsis> +</listitem> + +<listitem> +<cmdsynopsis> <command><link linkend="subscribe">subscribe</link></command> <arg choice="opt" rep="repeat"> <option>-group</option> diff -r 7a8ea1bb09f0 -r 9e876d64d3c8 globals.h --- a/globals.h Mon Jan 23 18:46:16 2017 -0800 +++ b/globals.h Mon Jan 23 19:01:50 2017 -0800 @@ -182,8 +182,9 @@ WHERE RX_LIST *UnMailLists INITVAL(0); WHERE RX_LIST *SubscribedLists INITVAL(0); WHERE RX_LIST *UnSubscribedLists INITVAL(0); -WHERE SPAM_LIST *SpamList INITVAL(0); +WHERE REPLACE_LIST *SpamList INITVAL(0); WHERE RX_LIST *NoSpamList INITVAL(0); +WHERE REPLACE_LIST *SubjectRxList INITVAL(0); /* bit vector for boolean variables */ diff -r 7a8ea1bb09f0 -r 9e876d64d3c8 hcache.c --- a/hcache.c Mon Jan 23 18:46:16 2017 -0800 +++ b/hcache.c Mon Jan 23 19:01:50 2017 -0800 @@ -1160,7 +1160,7 @@ unsigned int intval; } digest; struct md5_ctx ctx; - SPAM_LIST *spam; + REPLACE_LIST *spam; RX_LIST *nospam; hcachever = HCACHEVER; diff -r 7a8ea1bb09f0 -r 9e876d64d3c8 hdrline.c --- a/hdrline.c Mon Jan 23 18:46:16 2017 -0800 +++ b/hdrline.c Mon Jan 23 19:01:50 2017 -0800 @@ -199,6 +199,22 @@ return h->recipient; } +static char *apply_subject_mods (ENVELOPE *env) +{ + if (env == NULL) + return NULL; + + if (SubjectRxList == NULL) + return env->subject; + + if (env->subject == NULL || *env->subject == '\0') + return env->disp_subj = NULL; + + env->disp_subj = mutt_apply_replace(NULL, 0, env->subject, SubjectRxList); + return env->disp_subj; +} + + /* %a = address of author * %A = reply-to address (if present; otherwise: address of author * %b = filename of the originating folder @@ -566,20 +582,28 @@ break; case 's': - - if (flags & MUTT_FORMAT_TREE && !hdr->collapsed) { - if (flags & MUTT_FORMAT_FORCESUBJ) + char *subj; + if (hdr->env->disp_subj) + subj = hdr->env->disp_subj; + else if (SubjectRxList) + subj = apply_subject_mods(hdr->env); + else + subj = hdr->env->subject; + if (flags & MUTT_FORMAT_TREE && !hdr->collapsed) { - mutt_format_s (dest, destlen, "", NONULL (hdr->env->subject)); - snprintf (buf2, sizeof (buf2), "%s%s", hdr->tree, dest); - mutt_format_s_tree (dest, destlen, prefix, buf2); + if (flags & MUTT_FORMAT_FORCESUBJ) + { + mutt_format_s (dest, destlen, "", NONULL (subj)); + snprintf (buf2, sizeof (buf2), "%s%s", hdr->tree, dest); + mutt_format_s_tree (dest, destlen, prefix, buf2); + } + else + mutt_format_s_tree (dest, destlen, prefix, hdr->tree); } else - mutt_format_s_tree (dest, destlen, prefix, hdr->tree); + mutt_format_s (dest, destlen, prefix, NONULL (subj)); } - else - mutt_format_s (dest, destlen, prefix, NONULL (hdr->env->subject)); break; case 'S': diff -r 7a8ea1bb09f0 -r 9e876d64d3c8 init.c --- a/init.c Mon Jan 23 18:46:16 2017 -0800 +++ b/init.c Mon Jan 23 19:01:50 2017 -0800 @@ -453,11 +453,11 @@ return 0; } -static int remove_from_spam_list (SPAM_LIST **list, const char *pat); - -static int add_to_spam_list (SPAM_LIST **list, const char *pat, const char *templ, BUFFER *err) +static int remove_from_replace_list (REPLACE_LIST **list, const char *pat); + +static int add_to_replace_list (REPLACE_LIST **list, const char *pat, const char *templ, BUFFER *err) { - SPAM_LIST *t = NULL, *last = NULL; + REPLACE_LIST *t = NULL, *last = NULL; REGEXP *rx; int n; const char *p; @@ -490,12 +490,12 @@ break; } - /* If t is set, it's pointing into an extant SPAM_LIST* that we want to + /* If t is set, it's pointing into an extant REPLACE_LIST* that we want to * update. Otherwise we want to make a new one to link at the list's end. */ if (!t) { - t = mutt_new_spam_list(); + t = mutt_new_replace_list(); t->rx = rx; if (last) last->next = t; @@ -503,7 +503,7 @@ *list = t; } - /* Now t is the SPAM_LIST* that we want to modify. It is prepared. */ + /* Now t is the REPLACE_LIST* that we want to modify. It is prepared. */ t->template = safe_strdup(templ); /* Find highest match number in template string */ @@ -524,9 +524,9 @@ if (t->nmatch > t->rx->rx->re_nsub) { - snprintf (err->data, err->dsize, _("Not enough subexpressions for spam " + snprintf (err->data, err->dsize, _("Not enough subexpressions for " "template")); - remove_from_spam_list(list, pat); + remove_from_replace_list(list, pat); return -1; } @@ -535,38 +535,38 @@ return 0; } -static int remove_from_spam_list (SPAM_LIST **list, const char *pat) +static int remove_from_replace_list (REPLACE_LIST **list, const char *pat) { - SPAM_LIST *spam, *prev; + REPLACE_LIST *cur, *prev; int nremoved = 0; /* Being first is a special case. */ - spam = *list; - if (!spam) + cur = *list; + if (!cur) return 0; - if (spam->rx && !mutt_strcmp(spam->rx->pattern, pat)) + if (cur->rx && !mutt_strcmp(cur->rx->pattern, pat)) { - *list = spam->next; - mutt_free_regexp(&spam->rx); - FREE(&spam->template); - FREE(&spam); + *list = cur->next; + mutt_free_regexp(&cur->rx); + FREE(&cur->template); + FREE(&cur); return 1; } - prev = spam; - for (spam = prev->next; spam;) + prev = cur; + for (cur = prev->next; cur;) { - if (!mutt_strcmp(spam->rx->pattern, pat)) + if (!mutt_strcmp(cur->rx->pattern, pat)) { - prev->next = spam->next; - mutt_free_regexp(&spam->rx); - FREE(&spam->template); - FREE(&spam); - spam = prev->next; + prev->next = cur->next; + mutt_free_regexp(&cur->rx); + FREE(&cur->template); + FREE(&cur); + cur = prev->next; ++nremoved; } else - spam = spam->next; + cur = cur->next; } return nremoved; @@ -757,6 +757,88 @@ return 0; } +static int parse_replace_list (BUFFER *buf, BUFFER *s, unsigned long data, BUFFER *err) +{ + REPLACE_LIST **list = (REPLACE_LIST **)data; + BUFFER templ; + + memset(&templ, 0, sizeof(templ)); + + /* First token is a regexp. */ + if (!MoreArgs(s)) + { + strfcpy(err->data, _("not enough arguments"), err->dsize); + return -1; + } + mutt_extract_token(buf, s, 0); + + /* Second token is a replacement template */ + if (!MoreArgs(s)) + { + strfcpy(err->data, _("not enough arguments"), err->dsize); + return -1; + } + mutt_extract_token(&templ, s, 0); + + if (add_to_replace_list(list, buf->data, templ.data, err) != 0) { + FREE(&templ.data); + return -1; + } + FREE(&templ.data); + + return 0; +} + +static int parse_unreplace_list (BUFFER *buf, BUFFER *s, unsigned long data, BUFFER *err) +{ + REPLACE_LIST **list = (REPLACE_LIST **)data; + + /* First token is a regexp. */ + if (!MoreArgs(s)) + { + strfcpy(err->data, _("not enough arguments"), err->dsize); + return -1; + } + + mutt_extract_token(buf, s, 0); + remove_from_replace_list(list, buf->data); + return 0; +} + + +static void clear_subject_mods (void) +{ + int i; + if (Context && Context->msgcount) + { + for (i = 0; i < Context->msgcount; i++) + FREE(&Context->hdrs[i]->env->disp_subj); + } +} + + +static int parse_subjectrx_list (BUFFER *buf, BUFFER *s, unsigned long data, BUFFER *err) +{ + int rc; + + rc = parse_replace_list(buf, s, data, err); + if (rc == 0) + clear_subject_mods(); + return rc; +} + + +static int parse_unsubjectrx_list (BUFFER *buf, BUFFER *s, unsigned long data, BUFFER *err) +{ + int rc; + + rc = parse_unreplace_list(buf, s, data, err); + if (rc == 0) + clear_subject_mods(); + return rc; +} + + static int parse_spam_list (BUFFER *buf, BUFFER *s, unsigned long data, BUFFER *err) { BUFFER templ; @@ -785,7 +867,7 @@ mutt_extract_token (&templ, s, 0); /* Add to the spam list. */ - if (add_to_spam_list (&SpamList, buf->data, templ.data, err) != 0) { + if (add_to_replace_list (&SpamList, buf->data, templ.data, err) != 0) { FREE(&templ.data); return -1; } @@ -809,13 +891,13 @@ /* "*" is a special case. */ if (!mutt_strcmp(buf->data, "*")) { - mutt_free_spam_list (&SpamList); + mutt_free_replace_list (&SpamList); mutt_free_rx_list (&NoSpamList); return 0; } /* If it's on the spam list, just remove it. */ - if (remove_from_spam_list(&SpamList, buf->data) != 0) + if (remove_from_replace_list(&SpamList, buf->data) != 0) return 0; /* Otherwise, add it to the nospam list. */ diff -r 7a8ea1bb09f0 -r 9e876d64d3c8 init.h --- a/init.h Mon Jan 23 18:46:16 2017 -0800 +++ b/init.h Mon Jan 23 19:01:50 2017 -0800 @@ -3898,6 +3898,10 @@ static int parse_unattachments (BUFFER *, BUFFER *, unsigned long, BUFFER *); +static int parse_replace_list (BUFFER *, BUFFER *, unsigned long, BUFFER *); +static int parse_unreplace_list (BUFFER *, BUFFER *, unsigned long, BUFFER *); +static int parse_subjectrx_list (BUFFER *, BUFFER *, unsigned long, BUFFER *); +static int parse_unsubjectrx_list (BUFFER *, BUFFER *, unsigned long, BUFFER *); static int parse_alternates (BUFFER *, BUFFER *, unsigned long, BUFFER *); static int parse_unalternates (BUFFER *, BUFFER *, unsigned long, BUFFER *); @@ -3976,6 +3980,8 @@ { "spam", parse_spam_list, MUTT_SPAM }, { "nospam", parse_spam_list, MUTT_NOSPAM }, { "subscribe", parse_subscribe, 0 }, + { "subjectrx", parse_subjectrx_list, UL &SubjectRxList }, + { "unsubjectrx", parse_unsubjectrx_list, UL &SubjectRxList }, { "toggle", parse_set, MUTT_SET_INV }, { "unalias", parse_unalias, 0 }, { "unalternative_order",parse_unlist, UL &AlternativeOrderList }, diff -r 7a8ea1bb09f0 -r 9e876d64d3c8 mutt.h --- a/mutt.h Mon Jan 23 18:46:16 2017 -0800 +++ b/mutt.h Mon Jan 23 19:01:50 2017 -0800 @@ -569,20 +569,20 @@ struct rx_list_t *next; } RX_LIST; -typedef struct spam_list_t +typedef struct replace_list_t { REGEXP *rx; int nmatch; char *template; - struct spam_list_t *next; -} SPAM_LIST; + struct replace_list_t *next; +} REPLACE_LIST; #define mutt_new_list() safe_calloc (1, sizeof (LIST)) #define mutt_new_rx_list() safe_calloc (1, sizeof (RX_LIST)) -#define mutt_new_spam_list() safe_calloc (1, sizeof (SPAM_LIST)) +#define mutt_new_replace_list() safe_calloc (1, sizeof (REPLACE_LIST)) void mutt_free_list (LIST **); void mutt_free_rx_list (RX_LIST **); -void mutt_free_spam_list (SPAM_LIST **); +void mutt_free_replace_list (REPLACE_LIST **); LIST *mutt_copy_list (LIST *); int mutt_matches_ignore (const char *, LIST *); @@ -618,6 +618,7 @@ char *list_post; /* this stores a mailto URL, or nothing */ char *subject; char *real_subj; /* offset of the real subject */ + char *disp_subj; /* display subject (modified copy of subject) */ char *message_id; char *supersedes; char *date; diff -r 7a8ea1bb09f0 -r 9e876d64d3c8 muttlib.c --- a/muttlib.c Mon Jan 23 18:46:16 2017 -0800 +++ b/muttlib.c Mon Jan 23 19:01:50 2017 -0800 @@ -735,6 +735,7 @@ FREE (&(*p)->list_post); FREE (&(*p)->subject); /* real_subj is just an offset to subject and shouldn't be freed */ + FREE (&(*p)->disp_subj); FREE (&(*p)->message_id); FREE (&(*p)->supersedes); FREE (&(*p)->date); @@ -782,8 +783,10 @@ { base->subject = (*extra)->subject; base->real_subj = (*extra)->real_subj; + base->disp_subj = (*extra)->disp_subj; (*extra)->subject = NULL; (*extra)->real_subj = NULL; + (*extra)->disp_subj = NULL; } /* spam and user headers should never be hashed, and the new envelope may * have better values. Use new versions regardless. */ @@ -1074,6 +1077,98 @@ *p = '_'; } +/* Note this function uses a fixed size buffer of LONG_STRING and so + * should only be used for visual modifications, such as disp_subj. */ +char *mutt_apply_replace (char *dbuf, size_t dlen, char *sbuf, REPLACE_LIST *rlist) +{ + REPLACE_LIST *l; + static regmatch_t *pmatch = NULL; + static int nmatch = 0; + static char twinbuf[2][LONG_STRING]; + int switcher = 0; + char *p; + int i, n; + size_t cpysize, tlen; + char *src, *dst; + + if (dbuf && dlen) + dbuf[0] = '\0'; + + if (sbuf == NULL || *sbuf == '\0' || (dbuf && !dlen)) + return dbuf; + + twinbuf[0][0] = '\0'; + twinbuf[1][0] = '\0'; + src = twinbuf[switcher]; + dst = src; + + strfcpy(src, sbuf, LONG_STRING); + + for (l = rlist; l; l = l->next) + { + /* If this pattern needs more matches, expand pmatch. */ + if (l->nmatch > nmatch) + { + safe_realloc (&pmatch, l->nmatch * sizeof(regmatch_t)); + nmatch = l->nmatch; + } + + if (regexec (l->rx->rx, src, l->nmatch, pmatch, 0) == 0) + { + tlen = 0; + switcher ^= 1; + dst = twinbuf[switcher]; + + dprint (5, (debugfile, "mutt_apply_replace: %s matches %s\n", src, l->rx->pattern)); + + /* Copy into other twinbuf with substitutions */ + if (l->template) + { + for (p = l->template; *p && (tlen < LONG_STRING - 1); ) + { + if (*p == '%') + { + p++; + if (*p == 'L') + { + p++; + cpysize = MIN (pmatch[0].rm_so, LONG_STRING - tlen - 1); + strncpy(&dst[tlen], src, cpysize); + tlen += cpysize; + } + else if (*p == 'R') + { + p++; + cpysize = MIN (strlen (src) - pmatch[0].rm_eo, LONG_STRING - tlen - 1); + strncpy(&dst[tlen], &src[pmatch[0].rm_eo], cpysize); + tlen += cpysize; + } + else + { + n = strtoul(p, &p, 10); /* get subst number */ + while (isdigit((unsigned char)*p)) /* skip subst token */ + ++p; + for (i = pmatch[n].rm_so; (i < pmatch[n].rm_eo) && (tlen < LONG_STRING-1); i++) + dst[tlen++] = src[i]; + } + } + else + dst[tlen++] = *p++; + } + } + dst[tlen] = '\0'; + dprint (5, (debugfile, "mutt_apply_replace: subst %s\n", dst)); + } + src = dst; + } + + if (dbuf) + strfcpy(dbuf, dst, dlen); + else + dbuf = safe_strdup(dst); + return dbuf; +} + void mutt_FormatString (char *dest, /* output buffer */ size_t destlen, /* output buffer len */ @@ -1888,9 +1983,9 @@ } } -void mutt_free_spam_list (SPAM_LIST **list) +void mutt_free_replace_list (REPLACE_LIST **list) { - SPAM_LIST *p; + REPLACE_LIST *p; if (!list) return; while (*list) @@ -1926,7 +2021,7 @@ * * Returns 1 if the argument `s` matches a pattern in the spam list, otherwise * 0. */ -int mutt_match_spam_list (const char *s, SPAM_LIST *l, char *text, int textsize) +int mutt_match_spam_list (const char *s, REPLACE_LIST *l, char *text, int textsize) { static regmatch_t *pmatch = NULL; static int nmatch = 0; diff -r 7a8ea1bb09f0 -r 9e876d64d3c8 protos.h --- a/protos.h Mon Jan 23 18:46:16 2017 -0800 +++ b/protos.h Mon Jan 23 19:01:50 2017 -0800 @@ -271,6 +271,7 @@ void mutt_alias_delete_reverse (ALIAS *t); int mutt_alloc_color (int fg, int bg); int mutt_any_key_to_continue (const char *); +char *mutt_apply_replace (char *, size_t, char *, REPLACE_LIST *); int mutt_buffy_check (int); int mutt_buffy_notify (void); int mutt_builtin_editor (const char *, HEADER *, HEADER *); @@ -324,7 +325,7 @@ int mutt_link_threads (HEADER *, HEADER *, CONTEXT *); int mutt_lookup_mime_type (BODY *, const char *); int mutt_match_rx_list (const char *, RX_LIST *); -int mutt_match_spam_list (const char *, SPAM_LIST *, char *, int); +int mutt_match_spam_list (const char *, REPLACE_LIST *, char *, int); int mutt_messages_in_thread (CONTEXT *, HEADER *, int); int mutt_multi_choice (char *prompt, char *letters); int mutt_needs_mailcap (BODY *);
