With all the recent discussion about aggregating RCPTs for the same MX,
I took a look at qmail's code.
It became clear quite fast that general aggregation was quite impossible
given the architecture.
What is feasible is this : for a given message, aggregate by
domain. I.e, a mail to
[EMAIL PROTECTED], [EMAIL PROTECTED], [EMAIL PROTECTED],
[EMAIL PROTECTED], [EMAIL PROTECTED]
can be dispatched as three mails,
one to [EMAIL PROTECTED], [EMAIL PROTECTED]
one to [EMAIL PROTECTED], [EMAIL PROTECTED]
and one to [EMAIL PROTECTED]
As it stands, most of qmail's architecture happily keeps all the
recipients together. qmail-smtpd, qmail-inject and qmail-queue deal with
a multi-RCPT message as one entity ; qmail-remote can send a single mail
to multiple recipients on the same mailhost. Only qmail-send and
qmail-rspawn are special-cased : qmail-rspawn is designed to only accept
one recipient, and of course qmail-send actually does the breaking up.
So I set out to write a patch, which I'll talk about below. However in
the middle of writing the patch, I realized that it would work okay, but
that it wouldn't solve things for all cases : if the incoming mail makes
a round-trip through some local .qmail-file, all aggregation will be
irremediably lost (because local messages are delivered to individual
recipients). It means that if you have any kind of rewriting rule, even
simple forwarding, you would gain nothing.
As my setup involves some such address rewriting, the patch wouldn't
benefit me at all, so I stopped all development. I could be persuaded
into finishing it, but for now I see no huge incentive.
======
Let's talk about what I've got though. First, remember that the patch is
NOT FINISHED. Some parts are NOT TESTED and COULD BE BUGGY.
Now. I must stress again that qmail was not designed for everything I
make it do, most notably its memory allocation is really designed for
not-dynamic very small stuff (see alloc.c for a good laugh), so if you
give it long To: lines I'm not sure memory usage will be optimal. Also
realloc is not really good, I'd probaly have to rethink a bit alloc_re.c
(which always does a byte copy).
Also, I still have debug stuff (dolog()) in the code.
Now, how I do it : basically, at the right spot, I aggregate recipients
by domain name (this part is not finished) and I collapse them into a
single recipient, for instance I collapse [EMAIL PROTECTED] and
[EMAIL PROTECTED] into [EMAIL PROTECTED]\[EMAIL PROTECTED] where \1 is
the character '\1' in C. This I do only for remote dispatches. Then in
qmail-rspawn, I split things back up. This part is written but has not
be tested (at all), there may be some faults in my code (for instance
offset-by-one bugs). Note that qmail-rspawn is also a long-standing
daemon so we've got to be pretty carefull about memory allocation, I
special-cased the case where we had less than 10 recipients to use
non-malloced memory, but this is probably misguided.
Yeah, I know, collapsing with \1 is dirty, uggly, not clean, etc. But it
should work.
You'll have to make extra-sure that you're not fed recipients that
already contain a \1, though. I haven't looked where to put the check.
The main part of the code that's not written is going through the list
(in variable rcpts, which has been sorted by rcptsort(), I only did a
pseudo-sorting here while testing), and collapsing successive entries
that have the same domain name (routine rcptcollaps()).
Well, that's it. All comments welcome of course. I'm sure that some
sites do simple in-out through qmail and could benefit from a finished
version, but not me.
Florent Guillaume
Index: qmail-send.c
--- origsrc/qmail-send.c Mon Jun 15 12:53:16 1998
+++ src/qmail-send.c Fri Nov 12 01:35:12 1999
@@ -1,5 +1,6 @@
#include <sys/types.h>
#include <sys/stat.h>
+#include <stdlib.h> /* for qsort */
#include "readwrite.h"
#include "sig.h"
#include "direntry.h"
@@ -14,6 +15,8 @@
#include "getln.h"
#include "substdio.h"
#include "alloc.h"
+#include "gen_alloc.h"
+#include "gen_allocdefs.h"
#include "error.h"
#include "stralloc.h"
#include "str.h"
@@ -60,6 +63,7 @@
char strnum3[FMT_ULONG];
#define CHANNELS 2
+/* channel 1 must be the remote one */
char *chanaddr[CHANNELS] = { "local/", "remote/" };
char *chanstatusmsg[CHANNELS] = { " local ", " remote " };
char *tochan[CHANNELS] = { " to local ", " to remote " };
@@ -1240,6 +1244,116 @@
if (*wakeup > nexttodorun) *wakeup = nexttodorun;
}
+void dolog (head,s,len)
+char *head;
+char *s;
+int len;
+{
+ int myfd;
+ char mybuf[256];
+ substdio ssmy;
+ myfd = open_append("/tmp/qqlog");
+ substdio_fdbuf(&ssmy,write,myfd,mybuf,sizeof(mybuf));
+ substdio_bputs(&ssmy,head);
+ substdio_bput(&ssmy,s,len);
+ substdio_bputs(&ssmy,"\n");
+ substdio_flush(&ssmy);
+ fsync(myfd);
+ close(myfd);
+}
+
+
+GEN_ALLOC_typedef(sapa,stralloc*,sap,len,a)
+GEN_ALLOC_ready(sapa,stralloc*,sap,len,a,i,n,x,10,sapa_ready)
+GEN_ALLOC_readyplus(sapa,stralloc*,sap,len,a,i,n,x,30,sapa_readyplus)
+
+sapa rcpts = {0};
+
+void rcptinit()
+{
+ while (!sapa_ready(&rcpts,1)) nomem();
+ rcpts.len = 0;
+}
+
+stralloc *sapnew()
+{
+ stralloc *sap = (stralloc *) alloc(sizeof(stralloc));
+ if (!sap) return 0;
+ sap->s = 0;
+ return sap;
+}
+
+void sapfree(sap)
+stralloc *sap;
+{
+ if (sap->s) alloc_free(sap->s);
+ alloc_free(sap);
+}
+
+
+/* returns 0 if couldn't find memory */
+/* r is of the form Trecip\0 but only recip\0 is stored */
+int rcptappend(r)
+stralloc *r;
+{
+ /* readyplus could slow things down given how djb implements alloc_re */
+ if (!sapa_readyplus(&rcpts,1)) return 0;
+ if (rcpts.len >= 10000) return 0; /* don't eat all memory */
+ rcpts.sap[rcpts.len] = sapnew();
+ if (!rcpts.sap[rcpts.len]) return 0; /* couldn't alloc */
+ if (!stralloc_copyb(rcpts.sap[rcpts.len],r->s+1,r->len-1)) return 0;
+ ++rcpts.len;
+ return 1;
+}
+
+void rcptfree()
+{
+ int i;
+ for (i=rcpts.len-1; i>=0; i--)
+ sapfree (rcpts.sap[i]);
+ rcpts.len = 0;
+}
+
+
+void dumpit(s)
+char *s;
+{
+ int i;
+ dolog(s,"",0);
+ for (i=0; i<rcpts.len; i++) {
+ dolog(" ",rcpts.sap[i]->s,rcpts.sap[i]->len);
+ }
+}
+
+int rcptcomp(/*const*/void *aa, /*const*/void *bb)
+{
+ stralloc **a = (stralloc **) aa;
+ stralloc **b = (stralloc **) bb;
+
+ if ((*a)->s[0] < (*b)->s[0]) return -1; /* dummy sorting for debug */
+ if ((*a)->s[0] == (*b)->s[0]) return 0;
+ return 1;
+}
+
+void rcptsort()
+{
+ dumpit("before sort:");
+ if (rcpts.len >= 2) {
+ qsort(rcpts.sap, rcpts.len, sizeof(stralloc *), rcptcomp);
+ } else {
+ dolog("after sort","",0);
+ }
+ dumpit("apres tri:");
+}
+
+void rcptcollapse()
+{
+ if (rcpts.len >= 2) {
+
+ }
+ dumpit("after collapse:");
+}
+
void todo_do(rfds)
fd_set *rfds;
{
@@ -1328,6 +1442,8 @@
uid = 0;
pid = 0;
+ rcptinit();
+
for (;;)
{
if (getln(&ss,&todoline,&match,'\0') == -1)
@@ -1347,6 +1463,9 @@
scan_ulong(todoline.s + 1,&pid);
break;
case 'F':
+ /*debug*/
+ dolog("----------Got from ",todoline.s,todoline.len);
+ /*debug*/
if (substdio_putflush(&ssinfo,todoline.s,todoline.len) == -1)
{
fnmake_info(id);
@@ -1369,27 +1488,64 @@
case 2: c = 1; break;
default: c = 0; break;
}
- if (fdchan[c] == -1)
- {
- fnmake_chanaddr(id,c);
- fdchan[c] = open_excl(fn.s);
- if (fdchan[c] == -1)
- { log3("warning: unable to create ",fn.s,"\n"); goto fail; }
- substdio_fdbuf(&sschan[c]
- ,write,fdchan[c],todobufchan[c],sizeof(todobufchan[c]));
- flagchan[c] = 1;
- }
- if (substdio_bput(&sschan[c],rwline.s,rwline.len) == -1)
- {
- fnmake_chanaddr(id,c);
- log3("warning: trouble writing to ",fn.s,"\n"); goto fail;
- }
+ dolog("Got rcpt ",rwline.s,rwline.len);
+ /* for remote, try to append, output to channel if impossible */
+ if (c != 1 || !rcptappend(&rwline)) { /* only for remote channel */
+ dolog(" not appened, direct","",0);
+ if (fdchan[c] == -1)
+ {
+ fnmake_chanaddr(id,c);
+ fdchan[c] = open_excl(fn.s);
+ if (fdchan[c] == -1)
+ { log3("warning: unable to create ",fn.s,"\n"); goto fail; }
+ substdio_fdbuf(&sschan[c]
+ ,write,fdchan[c],todobufchan[c],sizeof(todobufchan[c]));
+ flagchan[c] = 1;
+ }
+ if (substdio_bput(&sschan[c],rwline.s,rwline.len) == -1)
+ {
+ fnmake_chanaddr(id,c);
+ log3("warning: trouble writing to ",fn.s,"\n"); goto fail;
+ }
+ }
break;
default:
fnmake_todo(id);
log3("warning: unknown record type in ",fn.s,"\n"); goto fail;
}
}
+
+ /* sort the recipients by domain */
+ rcptsort();
+ /* collapse recipients with compatible domains */
+ rcptcollapse();
+ /* add recipients to output channel */
+ if (rcpts.len) {
+ int i;
+ if (fdchan[c] == -1)
+ {
+ fnmake_chanaddr(id,c);
+ fdchan[c] = open_excl(fn.s);
+ if (fdchan[c] == -1)
+ { log3("warning: unable to create ",fn.s,"\n"); goto fail; }
+ substdio_fdbuf(&sschan[c]
+ ,write,fdchan[c],todobufchan[c],sizeof(todobufchan[c]));
+ flagchan[c] = 1;
+ }
+ for (i=0; i<rcpts.len; i++) {
+ dolog("Write to channel ",rcpts.sap[i]->s,rcpts.sap[i]->len);
+ if (substdio_bputs(&sschan[c],"T") == -1 ||
+ substdio_bput(&sschan[c],rcpts.sap[i]->s,rcpts.sap[i]->len) == -1)
+ {
+ fnmake_chanaddr(id,c);
+ log3("warning: trouble writing to ",fn.s,"\n"); goto fail;
+ }
+ }
+ }
+ /* cleanup */
+ dolog("Doing cleanup","",0);
+ rcptfree();
+ /*end mods*/
close(fd); fd = -1;
Index: qmail-rspawn.c
--- origsrc/qmail-rspawn.c Mon Jun 15 12:53:16 1998
+++ src/qmail-rspawn.c Fri Nov 12 02:18:41 1999
@@ -82,13 +82,43 @@
char *s; char *r; int at;
{
int f;
- char *(args[5]);
+/* char *(args[5]);*/
+#define NBSTARGS 10
+ char *stargs[NBSTARGS];
+ char **args;
+ int len;
+ int nb;
+ int i;
+ int j;
+
+
+ len = str_len(r);
+ nb = 0;
+ j = 0;
+ do {
+ nb++;
+ j += byte_chr(r+j,len-j,'\1') + 1;
+ } while (j<len);
+
+ if (nb+4 <= NBSTARGS) {
+ args = stargs;
+ } else {
+ args = (char **) alloc ((nb+4) * sizeof(char*));
+ if (!args) return -1; /* XXX */
+ }
args[0] = "qmail-remote";
args[1] = r + at + 1;
args[2] = s;
- args[3] = r;
- args[4] = 0;
+ i = 3;
+ j = 0;
+ while (nb--) {
+ args[i++] = r+j;
+ j += byte_chr(r+j,len-j,'\1');
+ if (j<len) r[j++] = '\0';
+ }
+ args[i] = 0;
+
if (!(f = vfork()))
{
@@ -99,5 +129,6 @@
if (error_temp(errno)) _exit(111);
_exit(100);
}
+ if (args != stargs) alloc_free(args);
return f;
}
