On Thu, 10 Jan 2013, David Lang wrote:
On Thu, 10 Jan 2013, Andrew Morgan wrote:
A corrupted seen file is the only thing that makes sense to me. If other
users can open the same folder, then the cyrus.header and cyrus.index files
must be sane.
As an experiment, you could move your seen file from lang.seen (or whatever
it's called) to lang.seen.bak. Then connect to IMAP as yourself and try to
open the folder. If it works, then it must have been a corrupted seen
file, and you can use skiplist.py to recover as much of it as possible.
Ok, the good news is that this seems to be the problem.
unfortunantly the skiplist recovery tool is not working.
# ./skiplist.py dlang.seen.bak >dlang.seen.txt
Traceback (most recent call last):
File "./skiplist.py", line 172, in <module>
values, keys = getkeys(fp)
File "./skiplist.py", line 152, in getkeys
spointer = unpack('>I', str_p)[0]
struct.error: unpack requires a string argument of length 4
# file dlang.seen.bak
dlang.seen.bak: Cyrus skiplist DB
I tried enabling debug mode in skiplist.py and I'm not seeing anything
different. This confuses me. I'm not that familiar with python, but as I read
the code, get_header() should be writing a bunch of stuff before it gets to
the getkeys() section that failing.
Hmmm, I haven't looked at the code in skiplist.py much. I have an older
version of skiplist.py, which I have attached to this email. Honestly, I
haven't used this since I upgraded to Cyrus v2.3.something. I think there
were some bugs in skiplist on the older versions. :)
Give the attached skiplist.py a shot! Worst case, you'll have to start
over with no Seen history. :(
Andy
#!/usr/bin/env python
# -*- Mode: Python; tab-width: 4 -*-
#
# Cyrus Imapd Skiplist db recovery tool
#
# Copyright (C) 2004 Gianluigi Tiesi <sherpya@xxxxxxxxxx>
# Copyright (C) 2004 NetFarm S.r.l. [http://www.netfarm.it]
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2, or (at your option) any later
# version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTIBILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
# ======================================================================
__version__= '0.1'
__doc__="""Cyrus skiplist db recover"""
from sys import argv,exit,stdout,stderr
from struct import unpack
from time import localtime, strftime
### User Conf
debug = 0
###
TIMEFMT ='%a, %d %b %Y %H:%M:%S %z'
PADDING = '\xff' * 4
INORDER = 1
ADD = 2
DELETE = 4
COMMIT = 255
DUMMY = 257
HEADER = -1
MAIN = -2
types = {
1: 'INORDER',
2: 'ADD',
4: 'DELETE',
255: 'COMMIT',
257: 'DUMMY',
-1: 'HEADER',
-2: '*'
}
def log(rtype, text):
global debug
if debug:
out = '[%s] %s\n' % (types[rtype], text)
stdout.write(out)
stdout.flush()
def roundto4(value):
if value % 4:
return ((value / 4) + 1) * 4
return value
def get_header(fp):
#### Magic ??
fp.seek(4)
sign = fp.read(16)
log(HEADER, sign[:-3])
version = unpack('>I', fp.read(4))[0]
version_minor = unpack('>I', fp.read(4))[0]
log(HEADER, 'Version %d,%d' % (version, version_minor))
maxlevel = unpack('>I', fp.read(4))[0]
curlevel = unpack('>I', fp.read(4))[0]
log(HEADER, 'Level %d/%d' % (curlevel, maxlevel))
listsize = unpack('>I', fp.read(4))[0]
log(HEADER, 'List size %d' % listsize)
logstart = unpack('>I', fp.read(4))[0]
log(HEADER, 'Offset %d' % logstart)
lastrecovery = localtime(unpack('>I', fp.read(4))[0])
lastrecovery = strftime(TIMEFMT, lastrecovery)
log(HEADER, 'Last Recovery %s' % lastrecovery)
return { 'version' : [version, version_minor],
'level' : [curlevel, maxlevel],
'listsize' : listsize,
'logstart' : logstart,
'lastrecover': lastrecovery
}
def getkeys(fp):
values = []
keys = {}
keystring = ''
datastring = ''
while 1:
log(MAIN, '-'*78)
stype = fp.read(4)
### EOF
if len(stype) != 4:
break
rtype = unpack('>I', stype)[0]
if not types.has_key(rtype):
log(MAIN, 'Invalid type %d' % rtype)
continue
log(rtype, 'Record type %s' % types[rtype])
if rtype == DELETE:
ptr = unpack('>I', fp.read(4))[0]
log(rtype, 'DELETE %d (0x%x)' % (ptr, ptr))
continue
if rtype == COMMIT:
continue
ksize = unpack('>I', fp.read(4))[0]
log(rtype, 'Key size %d (%d)' % (ksize, roundto4(ksize)))
if ksize:
keystring = fp.read(roundto4(ksize))[:ksize]
log(rtype, 'Key String %s' % keystring)
datasize = unpack('>I', fp.read(4))[0]
log(rtype, 'Data size %d (%d)' % (datasize, roundto4(datasize)))
if datasize:
datastring = fp.read(roundto4(datasize))[:datasize]
log(rtype, 'Data String %s' % datastring)
n = 0
while 1:
str_p = fp.read(4)
if str_p == PADDING:
break
spointer = unpack('>I', str_p)[0]
n = n +1
if spointer: log(rtype, 'Skip pointer %d' % spointer)
log(rtype, 'Total Skip pointers: %d' % n)
if rtype != DUMMY:
if keystring not in values:
values.append(keystring)
keys[keystring] = datastring
return values, keys
if __name__ == '__main__':
if len(argv) != 2:
print 'Usage: %s skiplist.file' % argv[0]
exit()
fp = open(argv[1], 'rb')
header = get_header(fp)
values, keys = getkeys(fp)
fp.close()
if debug: exit()
for v in values:
print '%s\t%s' % (v, keys[v])
----
Cyrus Home Page: http://www.cyrusimap.org/
List Archives/Info: http://lists.andrew.cmu.edu/pipermail/info-cyrus/
To Unsubscribe:
https://lists.andrew.cmu.edu/mailman/listinfo/info-cyrus
