Hi, I'm getting an assertion failed on the receiving side, causing syncs to fail for one user. The servers are setup so that only one is receiving any traffic other than replication at any time. The one that's only receiving replications is the one that's failing.
I've tried deleting the user's home on the receiving server, but it still
crashes during the sync. Oddly, the user's
home is 7.4G on the sending server, but ends up at 42G on the receiving side,
even after deleting and trying a fresh sync.
The mailbox implicated in the backtrace ("Spam") does have a very large number
of messages in it.
On sender:
Spam messages=1217764 recent=0 uidnext=1218103 uidvalidity=1379509105 unseen=16
highestmodseq=744588 vsize=34468460093
guid=090ed93a7a055559abf10200fdf6807a firstsaved=1498744186
On receiver:
Spam messages=1217766 recent=352 uidnext=1218105 uidvalidity=1379509105
unseen=16 highestmodseq=744589 vsize=34468496809
guid=090ed93a7a055559abf10200fdf6807a firstsaved=1519396172
Feb 23 14:57:20 dovecot: dsync-local(bgeels): Warning: fscking index file
/gnoc/mail/home/bgeels/mail/storage/dovecot.map.index
Feb 23 14:57:20 dovecot: dsync-local(bgeels): Warning: mdbox
/gnoc/mail/home/bgeels/mail/storage: rebuilding indexes
Feb 23 14:57:33 dovecot: dsync-local(bgeels): Panic: file mail-index-util.c:
line 10 (mail_index_uint32_to_offset):
assertion failed: (offset < 0x40000000)
Feb 23 14:57:33 dovecot: dsync-local(bgeels): Error: Raw backtrace:
/usr/lib64/dovecot/libdovecot.so.0(+0x9f3de)
[0x7feb584143de] -> /usr/lib64/dovecot/libdovecot.so.0(+0x9f4be)
[0x7feb584144be] ->
/usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7feb583a577c] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mail_index_uint32_to_offset+0xa0)
[0x7feb587906d0] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mail_cache_compress+0x854)
[0x7feb58774f34] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mail_index_sync_commit+0x25f)
[0x7feb587884ff] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mdbox_storage_rebuild_in_context+0x10de)
[0x7feb5870b3ae] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mdbox_sync_begin+0x858)
[0x7feb5870ccd8] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mdbox_sync+0x4c) [0x7feb5870ce7c] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mdbox_storage_sync_init+0x4b)
[0x7feb5870cf3b] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mailbox_sync_init+0x44)
[0x7feb586f2834] ->
/usr/lib64/dovecot/libdovecot-storage.so.0(mailbox_sync+0x37) [0x7feb586f28d7]
->
dovecot/doveadm-server(dsync_mailbox_import_deinit+0x475) [0x445495] ->
dovecot/doveadm-server() [0x43edc0] ->
dovecot/doveadm-server(dsync_brain_sync_mails+0x743) [0x43f653] ->
dovecot/doveadm-server(dsync_brain_run+0x541)
[0x43acf1] -> dovecot/doveadm-server() [0x43b070] -> dovecot/doveadm-server()
[0x44fe5f] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x52) [0x7feb58429cd2] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f)
[0x7feb5842b3bf] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x3c) [0x7feb58429d6c] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_run+0x38) [0x7feb58429f28] ->
dovecot/doveadm-server() [0x4209c5] ->
dovecot/doveadm-server() [0x422df6] -> dovecot/doveadm-server() [0x4377f4] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_call_io+0x52) [0x7feb58429cd2] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run_internal+0x10f)
[0x7feb5842b3bf] ->
/usr/lib64/dovecot/libdovecot.so.0(io_loop_handler_run+0x3c) [0x7feb58429d6c]
Feb 23 14:57:33 dsync-local(bgeels): Fatal: master: service(doveadm): child
82098 killed with signal 6 (core dumped)
I've attached the output of `doveconf -n` and the full backtrace from a core
dump.
Dovecot 2.2.33.2 (GhettoForge package)
CentOS 7 x86_64
XFS, no NFS.
-- Ian
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-100.el7_4.1 Copyright (C) 2013 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-redhat-linux-gnu". For bug reporting instructions, please see: <http://www.gnu.org/software/gdb/bugs/>... Reading symbols from /usr/bin/doveadm...(no debugging symbols found)...done. [New LWP 146391] Core was generated by `/usr/bin/doveadm dsync-server -ubgeels -U'. Program terminated with signal 6, Aborted. #0 0x00007fc20bc531f7 in raise () from /lib64/libc.so.6 Missing separate debuginfos, use: debuginfo-install dovecot22-2.2.33.2-2.gf.el7.x86_64 (gdb) #0 0x00007fc20bc531f7 in raise () from /lib64/libc.so.6 No symbol table info available. #1 0x00007fc20bc548e8 in abort () from /lib64/libc.so.6 No symbol table info available. #2 0x00007fc20c0803d6 in default_fatal_finish (type=<optimized out>, status=status@entry=0) at failures.c:201 backtrace = 0x1bcdfd0 "/usr/lib64/dovecot/libdovecot.so.0(+0x9f3de) [0x7fc20c0803de] -> /usr/lib64/dovecot/libdovecot.so.0(default_fatal_handler+0x2a) [0x7fc20c08044a] -> /usr/lib64/dovecot/libdovecot.so.0(i_fatal+0) [0x7fc"... #3 0x00007fc20c08044a in default_fatal_handler (ctx=0x7ffff51be4b0, format=<optimized out>, args=<optimized out>) at failures.c:215 status = 0 #4 0x00007fc20c01177c in i_panic (format=format@entry=0x7fc20c40ba20 "file %s: line %d (%s): assertion failed: (%s)") at failures.c:275 ctx = {type = LOG_TYPE_PANIC, exit_status = 0, timestamp = 0x0, timestamp_usecs = 0} args = {{gp_offset = 40, fp_offset = 48, overflow_arg_area = 0x7ffff51be5b0, reg_save_area = 0x7ffff51be4f0}} #5 0x00007fc20c3fc6d0 in mail_index_uint32_to_offset (offset=<optimized out>) at mail-index-util.c:10 __FUNCTION__ = "mail_index_uint32_to_offset" #6 0x00007fc20c3e0f34 in mail_cache_copy (ext_offsets=0x7ffff51be690, max_uid_r=0x7ffff51be64c, file_size_r=<synthetic pointer>, file_seq_r=<synthetic pointer>, fd=28, trans=0x1c48dd0, cache=<optimized out>) at mail-cache-compress.c:293 iter = {view = 0x1c19230, remap_counter = 2438427, seq = 1217766, rec = 0x7fc1f8deca18, pos = 832, rec_size = 832, offset = 0, trans_next_idx = 0, stop = 0, failed = 0, memory_appends_checked = 0, disk_appends_checked = 0} first_new_seq = 1 i = <optimized out> field = {field_idx = 30, size = 237, data = 0x7fc1f8decc68, offset = 1627311208} view = 0x1c1a620 hdr = {major_version = 1 '\001', compat_sizeof_uoff_t = 8 '\b', minor_version = 1 '\001', unused = 0 '\000', indexid = 1519395846, file_seq = 1519396173, continued_record_count = 0, record_count = 1217766, backwards_compat_used_file_size = 0, deleted_record_count = 0, field_header_offset = 0} output = 0x1c462e0 ext_offset = 1554079344 record_count = <optimized out> max_drop_time = <optimized out> ctx = {cache = 0x1c14790, buffer = 0x1c0eab0, field_seen = 0x1c0ea70, bitmask_pos = {arr = {buffer = 0x1bcdec0, element_size = 4}, v = 0x1bcdec0, v_modifiable = 0x1bcdec0}, field_file_map = 0x1bcde20, field_seen_value = 141 '\215', new_msg = true} cache_view = 0x1c19230 cache_rec = {prev_offset = 0, size = 876} message_count = 1217766 seq = <optimized out> used_fields_count = <optimized out> orig_fields_count = 33 idx_hdr = <optimized out> #7 mail_cache_compress_write (unlock=<synthetic pointer>, temp_path=0x1bcddd0 "/gnoc/mail/home/bgeels/mail/mailboxes/Spam/dbox-Mails/dovecot.index.cache.tmp", fd=28, trans=0x1c48dd0, cache=<optimized out>) at mail-cache-compress.c:339 file_seq = <optimized out> max_uid = 1218104 i = <optimized out> count = <optimized out> st = {st_dev = 25, st_ino = 1180560, st_nlink = 25, st_mode = 106699888, st_uid = 0, st_gid = 204982256, __pad0 = 32706, st_rdev = 140471401555808, st_size = 0, st_blksize = 1180560, st_blocks = 16, st_atim = {tv_sec = 29420496, tv_nsec = 140737305634904}, st_mtim = {tv_sec = 29779808, tv_nsec = 140737305634904}, st_ctim = {tv_sec = 29737888, tv_nsec = 140737305634904}, __unused = {29659536, 140737305634904, 29737888}} old_offset = 29443984 ext_offsets = {arr = {buffer = 0x1c1bc40, element_size = 4}, v = 0x1c1bc40, v_modifiable = 0x1c1bc40} offsets = <optimized out> file_size = <optimized out> #8 mail_cache_compress_locked (dotlock_r=0x7ffff51be668, unlock=<synthetic pointer>, trans=0x1c48dd0, cache=<optimized out>) at mail-cache-compress.c:478 data = 0x7fc197e00000 fd = 28 temp_path = 0x1bcddd0 "/gnoc/mail/home/bgeels/mail/mailboxes/Spam/dbox-Mails/dovecot.index.cache.tmp" #9 mail_cache_compress (cache=<optimized out>, trans=0x1c48dd0, lock_r=lock_r@entry=0x7ffff51be868) at mail-cache-compress.c:544 dotlock = 0x1c0ea00 unlock = true ret = <optimized out> __FUNCTION__ = "mail_cache_compress" #10 0x00007fc20c3f44ff in mail_index_sync_commit (_ctx=_ctx@entry=0x7ffff51be988) at mail-index-sync.c:860 ctx = 0x1c65540 index = 0x1c0aa30 cache_lock = 0x0 next_uid = 6 want_rotate = <optimized out> index_undeleted = false delete_index = <optimized out> ret = 0 ret2 = <optimized out> #11 0x00007fc20c3773ae in rebuild_mailbox (ns=<optimized out>, vname=<optimized out>, ctx=0x1c08250) at mdbox-storage-rebuild.c:588 box = 0x1c15df8 mbox = 0x1c15df8 rebuild_ctx = 0x0 ret = <optimized out> sync_ctx = 0x1c65540 view = 0x1c1a620 trans = 0x1c48dd0 error = <optimized out> #12 rebuild_namespace_mailboxes (ns=<optimized out>, ctx=0x1c08250) at mdbox-storage-rebuild.c:616 _data_stack_cur_id = 5 iter = 0x1c3a648 info = <optimized out> ret = 0 #13 rebuild_mailboxes (ctx=0x1c08250) at mdbox-storage-rebuild.c:636 storage = <optimized out> ns = <optimized out> #14 mdbox_storage_rebuild_scan (ctx=0x1c08250) at mdbox-storage-rebuild.c:948 data = 0x1c3bb50 data_size = 8 #15 mdbox_storage_rebuild_in_context (storage=0x1bf2238, atomic=atomic@entry=0x1c6c680) at mdbox-storage-rebuild.c:971 No locals. #16 0x00007fc20c378cd8 in mdbox_sync_begin (mbox=mbox@entry=0x1c1de98, flags=flags@entry=(unknown: 0), atomic=0x1c6c680, ctx_r=ctx_r@entry=0x7ffff51beba8) at mdbox-sync.c:245 storage = 0x1bf2238 hdr = <optimized out> ctx = <optimized out> reason = <optimized out> sync_flags = <optimized out> rebuild = true storage_rebuilt = false #17 0x00007fc20c378e7c in mdbox_sync (mbox=mbox@entry=0x1c1de98, flags=flags@entry=(unknown: 0)) at mdbox-sync.c:345 sync_ctx = 0x0 atomic = 0x1c6c680 ret = <optimized out> #18 0x00007fc20c378f3b in mdbox_storage_sync_init (box=0x1c1de98, flags=MAILBOX_SYNC_FLAG_FIX_INCONSISTENT) at mdbox-sync.c:368 mbox = 0x1c1de98 mdbox_sync_flags = (unknown: 0) ret = 0 #19 0x00007fc20c35e834 in mailbox_sync_init (box=0x1c1de98, flags=MAILBOX_SYNC_FLAG_FIX_INCONSISTENT) at mail-storage.c:1891 _data_stack_cur_id = 4 ctx = <optimized out> #20 0x00007fc20c35e8d7 in mailbox_sync (box=<optimized out>, flags=<optimized out>) at mail-storage.c:1947 ctx = 0x1c67a38 status = {sync_delayed_expunges = 0} #21 0x0000000000443415 in dsync_mailbox_import_deinit () No symbol table info available. #22 0x000000000043cd40 in dsync_brain_sync_half_finished.part.0 () No symbol table info available. #23 0x000000000043d5d3 in dsync_brain_sync_mails () No symbol table info available. #24 0x0000000000438c71 in dsync_brain_run () No symbol table info available. #25 0x0000000000438fb1 in dsync_brain_run_io () No symbol table info available. #26 0x000000000044dddf in dsync_ibc_stream_input () No symbol table info available. #27 0x00007fc20c095cd2 in io_loop_call_io (io=0x1bf8690) at ioloop.c:600 ioloop = 0x1bd3890 t_id = 2 __FUNCTION__ = "io_loop_call_io" #28 0x00007fc20c0973bf in io_loop_handler_run_internal (ioloop=ioloop@entry=0x1bd3890) at ioloop-epoll.c:223 ctx = 0x1bdf0d0 events = <optimized out> list = 0x1bf86f0 io = <optimized out> tv = {tv_sec = 4, tv_usec = 524014} events_count = <optimized out> msecs = <optimized out> ret = 1 i = 0 call = <optimized out> __FUNCTION__ = "io_loop_handler_run_internal" #29 0x00007fc20c095d6c in io_loop_handler_run (ioloop=ioloop@entry=0x1bd3890) at ioloop.c:649 No locals. #30 0x00007fc20c095f28 in io_loop_run (ioloop=0x1bd3890) at ioloop.c:624 __FUNCTION__ = "io_loop_run" #31 0x0000000000421d4e in cmd_dsync_server_run () No symbol table info available. #32 0x0000000000423706 in doveadm_mail_next_user () No symbol table info available. #33 0x0000000000424286 in doveadm_mail_cmd_exec () No symbol table info available. #34 0x0000000000424b3c in doveadm_mail_try_run () No symbol table info available. #35 0x0000000000414c65 in main () No symbol table info available. (gdb) quit
# 2.2.33.2 (d6601f4ec): /etc/dovecot/dovecot.conf
# Pigeonhole version 0.4.21 (92477967)
# OS: Linux 3.10.0-514.21.2.el7.x86_64 x86_64 CentOS Linux release 7.4.1708
(Core)
auth_cache_negative_ttl = 0
auth_cache_size = 10 M
auth_master_user_separator = *
auth_mechanisms = plain login
auth_username_format = %Ln
auth_verbose = yes
auth_worker_max_count = 100
base_dir = /var/run/dovecot/
default_client_limit = 2048
default_process_limit = 4096
default_vsz_limit = 1 G
dict {
acl = mysql:/etc/dovecot/dovecot-sql-dict.conf.ext
}
dsync_remote_cmd = ssh -l vmail -i /home/vmail/.ssh/id_rsa repl.peer doveadm
dsync-server -u%u -U
first_valid_uid = 1000
lda_mailbox_autocreate = yes
lda_mailbox_autosubscribe = yes
mail_attachment_dir = /gnoc/mail/attachments/
mail_attachment_fs = sis-queue /gnoc/mail/attachments/queue/:posix
mail_attachment_hash = %{sha256}
mail_gid = vmail
mail_location = mdbox:~/mail
mail_plugins = " listescape acl notify replication"
mail_uid = vmail
managesieve_notify_capability = mailto
managesieve_sieve_capability = fileinto reject envelope encoded-character
vacation subaddress comparator-i;ascii-numeric relational regex imap4flags copy
include variables body enotify environment mailbox date index ihave duplicate
mime foreverypart extracttext spamtest spamtestplus editheader
vnd.dovecot.debug imapsieve vnd.dovecot.imapsieve
mbox_write_locks = fcntl
mdbox_rotate_size = 20 M
namespace inbox {
inbox = yes
location =
mailbox Archive {
special_use = \Archive
}
mailbox Archives {
special_use = \Archive
}
mailbox Drafts {
auto = subscribe
special_use = \Drafts
}
mailbox Junk {
special_use = \Junk
}
mailbox Sent {
special_use = \Sent
}
mailbox "Sent Messages" {
special_use = \Sent
}
mailbox Spam {
special_use = \Junk
}
mailbox Trash {
auto = subscribe
special_use = \Trash
}
mailbox sent-mail {
special_use = \Sent
}
prefix =
separator = /
}
namespace sharedmail {
list = children
location = mdbox:%%h/mail:INDEXPVT=~/mail/shared/%%u
prefix = shared/%%u/
separator = /
subscriptions = no
type = shared
}
passdb {
args = /etc/dovecot/dovecot-sql-master.conf.ext
default_fields = userdb_master_user=%{login_user}
driver = sql
master = yes
pass = yes
}
passdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
plugin {
acl = vfile:/gnoc/mail/dict/acls:cache_secs=300
acl_shared_dict = proxy::acl
imapsieve_mailbox1_before = file:/etc/dovecot/sieve/report-spam.sieve
imapsieve_mailbox1_causes = COPY
imapsieve_mailbox1_name = Spam
imapsieve_mailbox2_before = file:/etc/dovecot/sieve/report-spam.sieve
imapsieve_mailbox2_causes = COPY
imapsieve_mailbox2_name = Junk
imapsieve_mailbox3_before = file:/etc/dovecot/sieve/report-ham.sieve
imapsieve_mailbox3_causes = COPY
imapsieve_mailbox3_from = Spam
imapsieve_mailbox3_name = *
imapsieve_mailbox4_before = file:/etc/dovecot/sieve/report-ham.sieve
imapsieve_mailbox4_causes = COPY
imapsieve_mailbox4_from = Junk
imapsieve_mailbox4_name = *
mail_log_events = delete undelete expunge copy mailbox_delete mailbox_rename
mail_log_fields = uid box msgid size
mail_replica = remote:vm...@repl.peer
sieve = ~/.dovecot.sieve
sieve_before = /etc/dovecot/sieve/before
sieve_before2 = /gnoc/mail/home/%Ln/sieve-before
sieve_dir = ~/sieve
sieve_extensions = +spamtest +spamtestplus +vnd.dovecot.debug +editheader
sieve_global_dir = /etc/dovecot/sieve/global
sieve_global_extensions = +vnd.dovecot.pipe
sieve_pipe_bin_dir = /etc/dovecot/sieve/bin
sieve_plugins = sieve_imapsieve sieve_extprograms
sieve_spamtest_max_header = X-Spam-Status: \w+,
score=-?[[:digit:]]+\.[[:digit:]]+ required=([[:digit:]]+\.[[:digit:]]+)
sieve_spamtest_status_header = X-Spam-Status: \w+,
score=(-?[[:digit:]]+\.[[:digit:]])
sieve_spamtest_status_type = score
}
protocols = imap pop3 lmtp sieve
service aggregator {
fifo_listener replication-notify-fifo {
mode = 0666
user = vmail
}
unix_listener replication-notify {
mode = 0666
user = vmail
}
}
service anvil {
client_limit = 16387
}
service auth {
client_limit = 327680
unix_listener /var/spool/postfix/private/auth {
group = postfix
mode = 0666
user = postfix
}
}
service dict {
unix_listener dict {
group = vmail
mode = 0660
user = vmail
}
}
service doveadm {
vsz_limit = 2 G
}
service imap {
executable = imap postlogin
process_limit = 4096
vsz_limit = 1 G
}
service lmtp {
unix_listener /var/spool/postfix/private/dovecot-lmtp {
group = postfix
mode = 0600
user = postfix
}
vsz_limit = 1 G
}
service managesieve-login {
inet_listener sieve {
port = 4190
}
}
service postlogin {
executable = script-login -d rawlog -I -t
}
service replicator {
process_min_avail = 1
unix_listener replicator-doveadm {
mode = 0666
}
}
ssl = required
ssl_ca = </etc/pki/tls/cert.pem
ssl_cert = </var/lib/acme/live/mail.ctc.grnoc.iu.edu/fullchain
ssl_cipher_list =
ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA
ssl_dh_parameters_length = 2048
ssl_key = # hidden, use -P to show it
ssl_prefer_server_ciphers = yes
ssl_protocols = !SSLv2 !SSLv3
ssl_require_crl = no
userdb {
args = /etc/dovecot/dovecot-sql.conf.ext
driver = sql
}
protocol lmtp {
mail_plugins = " listescape acl notify replication sieve"
postmaster_address = postmas...@globalnoc.iu.edu
}
protocol lda {
mail_plugins = " listescape acl notify replication sieve"
}
protocol imap {
mail_max_userip_connections = 256
mail_plugins = " listescape acl notify replication imap_acl imap_sieve"
}
smime.p7s
Description: S/MIME Cryptographic Signature
