The attached code is based on mbox2dbmail-fast.c from Mark Mackay, and
who knows before that.
I have added to it in an attempt to make a more complete tool for
transferring users from an mbox-style imap server (like UW-imap) to
dbmail. The program will conver a users inbox from the mail spool, and
optionally, convert their entire set of imap folders, creating an
identical folder tree. It preserves the seen and answered flags on all
messages.
To Build
Put the file in the dbmail source directory, and add these lines to the
Makefile (second line begins with a tab), then run 'make mb2db'.
mb2db: db.h auth.h dbmailtypes.h debug.h config.h $(INJECTOR_OBJECTS)
mb2db.c
$(CC) $(CFLAGS) mb2db.c -o mb2db $(UNIONE_OBJECTS) $(DIRS) $(LIB)
To Run
It should be run on your dbmail host. It reads the /etc/dbmail.conf file
for configuration. It needs access to the old imap server's mail spool
(for the inboxes), and the user's home directories (for their imap
folders).These can be copied, or NFS mounted, but you will need root
access to be able to read them (unless you run as the user). The mail
spool is expected at /var/spool/mail, but this can be changed in the
code by changing the spool variable and recompiling.
The dbmail user account must be created first using dbmail-adduser. Then run
mb2db <username> /home/<username>/mail
This expects to find the inbox in /var/spool/mail/<username>. The second
argument is optional. It should be the full path to the user's imap
folders. It will ignore files beginning with '.', and it will only
create a matching dbmail folder if it finds at least one message in an
mbox file. It is likely to crash if it encounters anything other than
mbox files.
Enjoy.
/*
* This program converts a user from an mbox style imap server (like UW-imap)
* to dbmail. It converts the inbox (from the mail spool) and optionally
* takes the path to the user's imap root folder directory and converts the
* full hierarchy of folders and messages.
*
* The dbmail account must already exist for the user.
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <sys/types.h>
#include <dirent.h>
#include <time.h>
#include <sys/stat.h>
#include <unistd.h>
#include "db.h"
#include "auth.h"
#include "dbmailtypes.h"
#include "debug.h"
#include "config.h"
#include <regex.h>
#define MAX_LINESIZE 1024
#define UID_SIZE 70
const char *mbox_delimiter_pattern = "^From .* ";
const char *spool = "/var/spool/mail/";
char blk[READ_BLOCK_SIZE + MAX_LINESIZE + 1];
/* syslog */
#define PNAME "dbmail/uw2db"
struct list sysItems; /* config item lists */
char *configFile = "/etc/dbmail.conf";
int processmbox (char *path, char *mailroot);
int process_mboxfile(char *file, u64_t userid, char *folder);
int recurse_folders(u64_t userid, char *topdir, char *topfolder);
int create_folder(u64_t userid, char *folder);
/* set up database login data */
extern field_t _db_host;
extern field_t _db_db;
extern field_t _db_user;
extern field_t _db_pass;
extern char *query;
int total_msgs = 0;
int main (int argc, char* argv[])
{
int result;
char *mailroot = "";
if (argc < 2)
{
printf ("Usage: mbox2dbmail <userid> [mailroot] \n");
printf (" userid = existing dbmail account name\n");
printf (" mailroot = optional full path to user's imap folder root.\n");
return -1;
}
if (argc == 3)
{
mailroot = argv[2];
}
openlog(PNAME, LOG_PID, LOG_MAIL); /* open connection to syslog */
configure_debug(TRACE_ERROR, 1, 0);
ReadConfig("DBMAIL", configFile, &sysItems);
GetDBParams(_db_host, _db_db, _db_user, _db_pass, &sysItems);
/* open dbase connections */
if (db_connect() < 0)
{
printf("Error opening dbase connections\n");
return -1;
}
result = processmbox(argv[1], mailroot);
printf("\n\nTotal mesages procesed: %d\n", total_msgs);
return result;
}
int processmbox (char *username, char *mailroot)
{
char path[256];
u64_t userid;
strcpy(path, spool);
strcat(path, username);
printf ("username %s\n", username);
printf("looking up existing user");
userid = auth_user_exists(username);
if (userid != -1 && userid != 0)
{
printf("Ok id [%llu]\n", userid);
fflush(stdout);
total_msgs += process_mboxfile(path, userid, "INBOX");
if (strlen(mailroot) > 0)
recurse_folders(userid, mailroot, "");
}
else
{
printf("user not found. Skipping\n");
}
return 0;
}
int recurse_folders(u64_t userid, char *topdir, char *topfolder)
{
char path[256];
char folder[256];
DIR *dir;
struct dirent *entry;
struct stat sbuf;
dir = opendir(topdir);
while((entry = readdir(dir)))
{
/* ignore . files */
if (strncmp(entry->d_name, ".", 1) == 0)
continue;
/* determine folder name */
strcpy(folder, topfolder);
if (strlen(topfolder) != 0)
strcat(folder, "/");
strcat(folder, entry->d_name);
strcpy(path, topdir);
strcat(path, "/");
strcat(path, entry->d_name);
stat(path, &sbuf);
if (sbuf.st_mode & S_IFDIR)
{
/* if a directory, recurse */
create_folder(userid, folder);
recurse_folders(userid, path, folder);
}
else
{
/* if a file, process it */
total_msgs += process_mboxfile(path, userid, folder);
}
}
return 0;
}
int create_folder(u64_t userid, char *folder)
{
if (db_findmailbox(folder, userid) == 0)
{
printf("\nCreating folder %s\n", folder);
db_createmailbox(folder, userid);
}
else
{
printf("\nFolder %s already exists\n", folder);
}
return 0;
}
int process_mboxfile(char *file, u64_t userid, char *folder)
{
regex_t preg;
int result;
FILE *infile;
int in_msg, header_passed=0;
char newunique[UID_SIZE];
unsigned cnt,len,newlines;
u64_t msgid=0, size, mailboxid;
char saved;
int msgcount = 0;
int flags[IMAP_NFLAGS] = {0,0,0,0,0,0};
if ((result = regcomp(&preg, mbox_delimiter_pattern, REG_NOSUB)) != 0)
{
trace(TRACE_ERROR,"Regex compilation failed.");
return -1;
}
if ( (infile = fopen(file, "r")) == 0)
{
trace(TRACE_ERROR,"Could not open file [%s]", infile);
return -1;
}
in_msg = 0;
cnt = 0;
size = 0;
newlines = 0;
while (!feof(infile) && !ferror(infile))
{
if (fgets(&blk[cnt], MAX_LINESIZE, infile) == 0)
break;
/* check if this is an mbox delimiter */
if (regexec(&preg, &blk[cnt], 0, NULL, 0) == 0)
{
if (!in_msg)
{
/* first header found - create folder */
create_folder(userid, folder);
in_msg = 1;
}
else
{
/* update & end message */
msgcount++;
printf("\rConverting %s :: %d", folder, msgcount);
db_insert_message_block(blk, cnt, msgid);
snprintf(newunique, UID_SIZE, "%lluA%lu", userid, time(NULL));
db_update_message(msgid, newunique, size+cnt, size+cnt+newlines);
trace(TRACE_ERROR, "message [%llu] inserted, [%u] bytes", msgid,
size+cnt);
/* move to correct folder */
mailboxid = db_get_mailboxid (userid, folder);
if (strcmp(folder, "INBOX") != 0)
{
snprintf (query, DEF_QUERYSIZE,
"UPDATE messages SET mailbox_idnr = %llu "
"where message_idnr=%llu",
mailboxid, msgid);
if (db_query (query)==-1)
{
trace(TRACE_STOP,"Move folders failed: dbquery failed");
}
}
/* set seen and answered flags */
db_set_msgflag(msgid, mailboxid, flags, IMAPFA_ADD);
}
/* start new message */
msgid = db_insert_message(userid, 0, 0);
header_passed = 0;
cnt = 0;
size = 0;
newlines = 0;
flags[0] = 0;
flags[1] = 0;
}
else
{
newlines++;
if (header_passed == 0)
{
/* we're still reading the header */
len = strlen(&blk[cnt]);
if (strcmp(&blk[cnt], "Status: RO\n") == 0)
{
flags[0] = 1;
}
if (strcmp(&blk[cnt], "X-Status: A\n") == 0)
{
flags[1] = 1;
}
if (strcmp(&blk[cnt], "\n") == 0)
{
db_insert_message_block(blk, cnt+len, msgid);
header_passed = 1;
size += (cnt+len);
cnt = 0;
}
else
cnt += len;
}
else
{
/* this is body data */
len = strlen(&blk[cnt]);
cnt += len;
if (cnt >= READ_BLOCK_SIZE)
{
/* write block */
saved = blk[READ_BLOCK_SIZE];
blk[READ_BLOCK_SIZE] = '\0';
db_insert_message_block(blk, READ_BLOCK_SIZE, msgid);
blk[READ_BLOCK_SIZE] = saved;
memmove(blk, &blk[READ_BLOCK_SIZE], cnt - (READ_BLOCK_SIZE));
size += READ_BLOCK_SIZE;
cnt -= READ_BLOCK_SIZE;
}
}
}
}
/* update & end message */
if (msgid > 0)
{
db_insert_message_block(blk, cnt, msgid);
snprintf(newunique, UID_SIZE, "%lluA%lu", userid, time(NULL));
db_update_message(msgid, newunique, size+cnt, size+cnt+newlines);
trace(TRACE_ERROR, "message [%llu] inserted, [%u] bytes", msgid,
size+cnt);
/* Set flags and move to correct folder */
mailboxid = db_get_mailboxid (userid, folder);
if (strcmp(folder, "INBOX") != 0)
{
snprintf (query, DEF_QUERYSIZE,
"UPDATE messages SET mailbox_idnr = %llu "
"where message_idnr=%llu",
mailboxid, msgid);
if (db_query (query)==-1)
{
trace(TRACE_STOP,"Move folders failed: dbquery failed");
}
}
db_set_msgflag(msgid, mailboxid, flags, IMAPFA_ADD);
msgcount++;
}
fclose(infile);
printf("\rConverting %s :: %d", folder, msgcount);
return msgcount;
}
