On 9. Mar 2021, at 23.26, Christopher Wensink <cwens...@five-star-plastics.com> wrote:
Looks like the index file is a bit broken for whatever reason. You could delete it (which would lose flags if you're using dbox) or you could use the attached script to run: dovecot-index-log.py --file /path/to/broken/folder/dovecot.index.log --ext_hdr_shrink --add_ext_intro hdr-vsize --add_ext_hdr 00000000000000000000000000000000 Of course at some point we'll need to fix the code to fix it automatically.. |
#!/usr/bin/env python3 from struct import unpack, unpack_from, pack from collections import namedtuple from optparse import OptionParser import binascii
def pad4(payload):
pad = 4 - (len(payload) % 4)
if pad == 4:
return payload
return payload.ljust(len(payload)+pad, b'\x00')
class DovecotIndexSect(object):
ExtIntroHeader = namedtuple("IntroHeader", "ext_id reset_id hdr_size record_size record_align flags name_size name")
KIND_TO_TEXT = {
0x00000001 | 0xcd90: "EXPUNGE",
0x00000002: "APPEND",
0x00000004: "FLAG_UPDATE",
0x00000020: "HEADER_UPDATE",
0x00000040: "EXT_INTRO",
0x00000080: "EXT_RESET",
0x00000100: "EXT_HDR_UPDATE",
0x00000200: "EXT_REC_UPDATE",
0x00000400: "KEYWORD_UPDATE",
0x00000800: "KEYWORD_RESET",
0x00001000: "EXT_ATOMIC_INC",
0x00002000 | 0xcd90: "EXPUNGE_GUID",
0x00008000: "MODSEQ_UPDATE",
0x00010000: "EXT_HDR_UPDATE32",
0x00020000: "INDEX_DELETED",
0x00040000: "INDEX_UNDELETED",
0x00080000: "BOUNDARY",
0x00100000: "ATTRIBUTE_UPDATE",
}
def __init__(self, log, kind):
self.log = log
self.kind = kind
self.mod = kind & ~(0x0fffffff)
self.type = kind & 0x0fffffff
self.ext_id = 0
self.name = ""
self.str = ""
def __str__(self):
return self.KIND_TO_TEXT[self.type] + " " + self.str
def parse_EXPUNGE(self):
(self.uid1, self.uid2) = unpack_from("II", self.data)
def ext_get_name(self):
if self.ext_id > 0 and len(self.log.ext_names) >= self.ext_id:
self.name = self.log.ext_names[self.ext_id-1]
else:
self.name = "<missing>"
def parse_EXT_INTRO(self):
self.hdr = self.ExtIntroHeader._make(unpack_from("IIIHHHH", self.data) + ("",))
name = self.data[20:20+self.hdr.name_size].decode()
self.hdr = self.hdr._replace(name = name)
self.ext_id = self.hdr.ext_id
if self.ext_id == 4294967295:
self.ext_id = -1
self.name = name
if self.ext_id < 0:
if name in self.log.ext_names:
self.ext_id = self.log.ext_names.index(name)+1
else:
self.ext_id = len(self.log.ext_names)+1
self.log.ext_names += [name]
elif self.ext_id > 0:
self.ext_get_name()
self.log.last_ext_id = self.ext_id
self.str = "(name=%s, ext_id=%u)" % (self.name, self.ext_id)
def parse_EXT_HDR_UPDATE32(self):
self.ext_id = self.log.last_ext_id
self.ext_get_name()
(self.offset, self.len) = unpack_from("II", self.data)
self.str = "(name=%s, ext_id=%u, offset=%u, len=%u, data=%s" % (self.name,self.ext_id,self.offset,self.len,binascii.hexlify(self.data[8:8+self.len]).decode())
def parse_EXT_HDR_UPDATE(self):
self.ext_id = self.log.last_ext_id
self.ext_get_name()
(self.offset, self.len) = unpack_from("HH", self.data)
self.str = "(name=%s, ext_id=%u, offset=%u, len=%u, data=%s)" % (self.name,self.ext_id,self.offset,self.len,binascii.hexlify(self.data[4:4+self.len]).decode())
def parse_EXT_REC_UPDATE(self):
self.ext_id = self.log.last_ext_id
self.ext_get_name()
(self.uid,) = unpack_from("I", self.data)
self.str = "(name=%s, ext_id=%u, uid=%u)" % (self.name,self.ext_id,self.uid)
def parse(self):
meth = "parse_%s" % (self.KIND_TO_TEXT[self.type])
meth = getattr(self, "parse_%s" % (self.KIND_TO_TEXT[self.type]), None)
if meth and callable(meth):
meth()
class DovecotIndexLog(object):
LogHeader = namedtuple("LogHeader", "major minor hdr_size indexid file_seq prev_file_seq prev_file_offset create_stamp initial_modseq compat_flags unused unused2")
def __init__(self, fn):
self.fn = fn
self.ext_names = []
self.sections = {}
self.parse()
def uint32_to_offset(self, offset):
offset >>= 2
offset = 0x00000080 | (offset & 0x0000007f) | 0x00008000 | ((offset & 0x00003f80) >> 7 << 8) | 0x00800000 | ((offset & 0x001fc000) >> 14 << 16) | 0x80000000 | ((offset & 0x0fe00000) >> 21 << 24)
return offset
def offset_to_uint32(self, offset):
if ((offset & 0x80808080) != 0x80808080):
return 0
return ((offset & 0x0000007f) |
((offset & 0x00007f00) >> 8 << 7) |
((offset & 0x007f0000) >> 16 << 14) |
((offset & 0x7f000000) >> 24 << 21)) << 2
def parse(self):
with open(self.fn, "rb") as f:
offset = 0
self.hdr = f.read(40)
self.header = self.LogHeader._make(unpack("BBHIIIIIQB3sI", self.hdr))
# parse sections
while True:
offset = f.tell()
try:
(size,) = unpack(">I", f.read(4))
except:
break
(kind,) = unpack("I", f.read(4))
size = self.offset_to_uint32(size)
sect = DovecotIndexSect(self, kind)
sect.size = size
sect.data = f.read(size-8)
sect.parse()
self.sections[offset] = sect
def _print(self):
print(self.header)
for offset in sorted(self.sections):
print("%04x: %s" % (offset, self.sections[offset]))
def write(self, fn):
with open(fn, "wb") as f:
f.write(self.hdr)
for idx in sorted(self.sections):
sect = self.sections[idx]
size = self.uint32_to_offset(sect.size)
f.write(pack(">I", size))
f.write(pack("I", sect.kind))
f.write(sect.data)
def drop(self, hdr):
self.sections = {k: v for k, v in self.sections.items() if v.name != prog.drop}
def add_ext_intro(self, hdr, hdr_size=0, rec_size=0, shrink=False):
offset = max(self.sections)
sect = DovecotIndexSect(self, 0x10000040)
if shrink:
flags = 0
else:
flags = 1 # MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK
sect.data = pack("IIIHHHH", 4294967295, 0, hdr_size, rec_size, 1, flags, len(hdr))
sect.data += pad4(hdr.encode())
sect.size = len(sect.data)+8
self.sections[offset+1] = sect
def add_ext_hdr(self, payload):
offset = max(self.sections)
sect = DovecotIndexSect(self, 0x10010000)
sect.data = pack("II", 0, len(payload))
sect.data += pad4(payload)
sect.size = len(sect.data)+8
self.sections[offset+1] = sect
def add_ext_rec(self, uid, payload):
offset = max(self.sections)
sect = DovecotIndexSect(self, 0x10000200)
sect.data = pack("I", int(uid))
sect.data += pad4(payload)
sect.size = len(sect.data)+8
self.sections[offset+1] = sect
def add_expunge(self, uid):
offset = max(self.sections)
sect = DovecotIndexSect(self, 0x00000001 | 0xcd90)
sect.data = pack("II", int(uid), int(uid))
sect.size = len(sect.data)+8
self.sections[offset+1] = sect
def read_options():
parser = OptionParser()
parser.add_option("", "--file", dest="filename",
help="index.log file to read", metavar="FILE")
parser.add_option("", "--print", dest="_print",
action="store_true", default=False,
help="dump the index.log file contents")
parser.add_option("", "--drop", dest="drop",
help="drop named header extension", metavar="NAME")
parser.add_option("", "--add_ext_intro", dest="add_ext_intro",
help="add named extension introduction", metavar="NAME")
parser.add_option("", "--add_ext_hdr", dest="add_ext_hdr",
help="add extension header for previous ext intro", metavar="PAYLOAD")
parser.add_option("", "--ext_hdr_shrink", dest="ext_hdr_shrink",
action="store_true", default=False,
help="shrink the extension header to specified header size")
parser.add_option("", "--add_ext_rec", dest="add_ext_rec",
help="add extension record for previous ext intro", metavar="PAYLOAD")
parser.add_option("", "--expunge", dest="expunge_uid",
help="expunge given UID range", metavar="UID")
parser.add_option("", "--payload", dest="payload",
help="payload to add", metavar="HEXDATA", default="")
(options, args) = parser.parse_args()
return options
if __name__ == "__main__":
prog = read_options()
log = DovecotIndexLog(prog.filename)
ext_hdr_len = 0
if prog.add_ext_hdr:
prog.add_ext_hdr = binascii.unhexlify(prog.add_ext_hdr)
ext_hdr_len = len(prog.add_ext_hdr)
ext_rec_len = 0
if prog.add_ext_rec and prog.payload:
prog.payload = binascii.unhexlify(prog.payload)
ext_rec_len = len(prog.payload)
if prog.drop:
log.drop(prog.drop)
log.write(prog.filename)
if prog.add_ext_intro:
log.add_ext_intro(prog.add_ext_intro, ext_hdr_len, ext_rec_len, prog.ext_hdr_shrink)
log.write(prog.filename)
if prog.add_ext_hdr:
log.add_ext_hdr(prog.add_ext_hdr)
log.write(prog.filename)
if prog.add_ext_rec:
log.add_ext_rec(prog.add_ext_rec, prog.payload)
log.write(prog.filename)
if prog.expunge_uid:
log.add_expunge(prog.expunge_uid)
log.write(prog.filename)
if prog._print == True:
log._print()
