On Tuesday 12 January 2010 18:15:29 Bron Gondwana wrote:
> On Tue, Jan 12, 2010 at 08:57:28AM -0800, wcooley@xxxxxxxxxxx wrote:
> >
> > Perhaps it would be a good idea, then, to make the '-k' behavior the
> > default and use another option to invert the logic?
>
> Absolutely... it will probably happen in 2.4 or 2.5... it's on my list
> of incompatible changes!
If anyone is interested, here's a patch that we use that turns on -k by
default when delayed expunge is enabled. We've never seen a need for the "non
-k" behavior since there are other ways to achieve the same results, if you
really do need to delete messages & meta data. Of course this patch is a lot
larger than it needs to be for just the -k change, since it adds a -w
(warning) and -e (add un-indexed files to expunge). Also, it looks like a lot
of changes, but it's mostly additional if statements and some corresponding
formatting fixes.
-Brian
commit 989d72de5762dcc0962d0919f8c4652798816cf4
Author: Brian Awood <bawood@xxxxxxxxx>
Date: Mon Aug 10 17:07:02 2009 -0400
Add -w and -e options to reconstruct
Added -w to warn admins about potential user visible changes, like
an increase or decrease in the number of messages indexed in a
mailbox.
Added -e to enable reconstructing an expunge file. Using -e will put
all unindexed files into the expunge file.
This patch also enables the -k behavior with delayed expunge is
enabled.
diff --git a/imap/reconstruct.c b/imap/reconstruct.c
index 6a115e0..23be7e9 100644
--- a/imap/reconstruct.c
+++ b/imap/reconstruct.c
@@ -134,10 +134,13 @@ struct uniqmailid * find_uniqid (char * mailboxname, char * mailboxid);
extern cyrus_acl_canonproc_t mboxlist_ensureOwnerRights;
int code = 0;
+int status = 0;
int keepflag = 0;
int syncflag = 0;
int guid_clear = 0;
int guid_set = 0;
+int warn_only = 0;
+int add_expunge = 0;
int main(int argc, char **argv)
{
@@ -163,7 +166,7 @@ int main(int argc, char **argv)
assert(INDEX_HEADER_SIZE == (OFFSET_SPARE4+4));
assert(INDEX_RECORD_SIZE == (OFFSET_MODSEQ+4));
- while ((opt = getopt(argc, argv, "C:kp:rmfsxgG")) != EOF) {
+ while ((opt = getopt(argc, argv, "C:kp:rmfsxgGwe")) != EOF) {
switch (opt) {
case 'C': /* alt config file */
alt_config = optarg;
@@ -205,6 +208,14 @@ int main(int argc, char **argv)
guid_set = 1;
break;
+ case 'w':
+ warn_only = 1;
+ break;
+
+ case 'e':
+ add_expunge = 1;
+ break;
+
default:
usage();
}
@@ -213,6 +224,11 @@ int main(int argc, char **argv)
cyrus_init(alt_config, "reconstruct", 0);
global_sasl_init(1,0,NULL);
+ if( config_getenum(IMAPOPT_EXPUNGE_MODE) ==
+ IMAP_ENUM_EXPUNGE_MODE_DELAYED ) {
+ keepflag = 1;
+ }
+
/* Set namespace -- force standard (internal) */
if ((r = mboxname_init_namespace(&recon_namespace, 1)) != 0) {
syslog(LOG_ERR, error_message(r));
@@ -302,10 +318,14 @@ int main(int argc, char **argv)
(*recon_namespace.mboxname_tointernal)(&recon_namespace, argv[i],
NULL, buf);
- r = mboxlist_createmailbox(buf, 0, start_part, 1,
+ if( warn_only ) {
+ printf( "Add to mailbox list: %s\n", buf );
+ } else {
+ r = mboxlist_createmailbox(buf, 0, start_part, 1,
"cyrus", NULL, 0, 0, !xflag);
- if(r) {
- fprintf(stderr, "could not create %s\n", argv[i]);
+ if(r) {
+ fprintf(stderr, "could not create %s\n", argv[i]);
+ }
}
}
}
@@ -363,10 +383,14 @@ int main(int argc, char **argv)
p = head.next;
head.next = p->next;
- /* create p (database only) and reconstruct it */
- /* partition is defined by the parent mailbox */
- r = mboxlist_createmailbox(p->name, 0, NULL, 1,
+ if( warn_only ) {
+ printf( "Discovered mailbox %s\n", p->name );
+ } else {
+ /* create p (database only) and reconstruct it */
+ /* partition is defined by the parent mailbox */
+ r = mboxlist_createmailbox(p->name, 0, NULL, 1,
"cyrus", NULL, 0, 0, !xflag);
+ }
if (!r) {
do_reconstruct(p->name, strlen(p->name), 0, &head);
} else {
@@ -387,7 +411,8 @@ int main(int argc, char **argv)
cyrus_done();
- return code;
+ if(code) return code;
+ return ( status ? 1: 0);
}
void usage(void)
@@ -833,11 +858,16 @@ int reconstruct(char *name, struct discovered *found)
snprintf(mbpath, sizeof(mbpath), "%s%s", path, FNAME_HEADER);
if(stat(mbpath, &sbuf) == -1) {
/* Header doesn't exist, create it! */
- r = mailbox_create(name, mypart, myacl, NULL,
+ if( warn_only ) {
+ printf( "stat() error of header file, does it exist?\n" );
+ return IMAP_IOERROR;
+ } else {
+ r = mailbox_create(name, mypart, myacl, NULL,
((mytype & MBTYPE_NETNEWS) ?
MAILBOX_FORMAT_NETNEWS :
MAILBOX_FORMAT_NORMAL), NULL);
- if(r) return r;
+ if(r) return r;
+ }
}
/* Now open just the header (it will hopefully be valid) */
@@ -859,8 +889,7 @@ int reconstruct(char *name, struct discovered *found)
if (mailbox.quota.root) free(mailbox.quota.root);
if (hasquota) {
mailbox.quota.root = xstrdup(quota_root);
- }
- else {
+ } else {
mailbox.quota.root = 0;
}
@@ -893,7 +922,11 @@ int reconstruct(char *name, struct discovered *found)
}
if(strcmp(list_acl, mailbox.acl)) {
- r = mboxlist_update(name, list_type, list_part, mailbox.acl, 0);
+ if( warn_only ) {
+ printf( "Update mailbox list acl: %s, %s\n", name, mailbox.acl );
+ } else {
+ r = mboxlist_update(name, list_type, list_part, mailbox.acl, 0);
+ }
}
if (r) {
mailbox_close(&mailbox);
@@ -931,12 +964,16 @@ int reconstruct(char *name, struct discovered *found)
map_free(&expunge_base, &expunge_len);
close(expunge_fd);
expunge_fd = -1;
- syslog(LOG_ERR, "Unable to verify expunge header - deleting: %s",
+ if( warn_only ) {
+ printf( "Bad expunge header: %s\n", name );
+ } else {
+ syslog(LOG_ERR, "Unable to verify expunge header - deleting: %s",
mailbox.name);
- reconstruct_delete_single(&mailbox,
+ reconstruct_delete_single(&mailbox,
IMAP_ENUM_METAPARTITION_FILES_EXPUNGE,
FNAME_EXPUNGE_INDEX, NULL);
- }
+ }
+ }
}
if (expunge_base && (expunge_len >= INDEX_HEADER_SIZE))
expunge_exists = ntohl(*((bit32 *)(expunge_base+OFFSET_EXISTS)));
@@ -964,9 +1001,15 @@ int reconstruct(char *name, struct discovered *found)
}
if (!keepflag && (expunge_exists > 0))
+ if( warn_only ) {
+ printf( "Remove from %s, %i expunged messages!\n", name,
+ expunge_exists );
+ } else {
reconstruct_clear_expunged(&mailbox, expunge_uidmap, expunge_exists);
+ }
/* Create new index/cache/expunge files */
+ if( !warn_only ) {
reconstruct_make_path(fnamebuf, sizeof(fnamebuf), &mailbox,
IMAP_ENUM_METAPARTITION_FILES_INDEX,
FNAME_INDEX, ".NEW");
@@ -995,6 +1038,7 @@ int reconstruct(char *name, struct discovered *found)
fwrite(buf, 1, INDEX_HEADER_SIZE, newindex);
fwrite(buf, 1, INDEX_HEADER_SIZE, newexpunge);
retry_write(newcache_fd, buf, sizeof(bit32));
+ }
/* Find all message files in directory */
uid = (unsigned long *) xmalloc(UIDGROW * sizeof(unsigned long));
@@ -1063,7 +1107,11 @@ int reconstruct(char *name, struct discovered *found)
if (sbuf.st_size == 0) {
/* Zero-length message file--blow it away */
fclose(msgfile);
+ if( warn_only ) {
+ printf( "Delete Zero-length message file: %s\n", msgfname );
+ } else {
unlink(msgfname);
+ }
continue;
}
@@ -1139,8 +1187,9 @@ int reconstruct(char *name, struct discovered *found)
/* NB: message_create_record() will reconstruct GUID if NULL */
if (((r = message_parse_file(msgfile, NULL, NULL, &body)) != 0) ||
- ((r = message_create_record(mailbox.name, newcache_fd,
- &message_index, body)) != 0)) {
+ ( !warn_only &&
+ ( r = message_create_record(mailbox.name, newcache_fd,
+ &message_index, body) != 0 ))) {
r = IMAP_IOERROR;
goto bail;
}
@@ -1150,19 +1199,23 @@ int reconstruct(char *name, struct discovered *found)
/* Clear out existing or regenerated GUID */
if (guid_clear) message_guid_set_null(&message_index.guid);
- if (expunge_found && keepflag) {
+ if ((expunge_found && keepflag) || ( !index_found && add_expunge)) {
/* Write out new entry in expunge file */
reconstruct_counts_update(&expunge_counts, &message_index);
- mailbox_index_record_to_buf(&message_index, buf);
- n = fwrite(buf, 1, INDEX_RECORD_SIZE, newexpunge);
+ if( !warn_only ) {
+ mailbox_index_record_to_buf(&message_index, buf);
+ n = fwrite(buf, 1, INDEX_RECORD_SIZE, newexpunge);
+ }
} else {
/* Write out new entry in index file */
reconstruct_counts_update(&index_counts, &message_index);
- mailbox_index_record_to_buf(&message_index, buf);
- n = fwrite(buf, 1, INDEX_RECORD_SIZE, newindex);
+ if( !warn_only ) {
+ mailbox_index_record_to_buf(&message_index, buf);
+ n = fwrite(buf, 1, INDEX_RECORD_SIZE, newindex);
+ }
}
- if (n != INDEX_RECORD_SIZE) {
+ if (!warn_only && n != INDEX_RECORD_SIZE) {
r = IMAP_IOERROR;
goto bail;
}
@@ -1201,33 +1254,46 @@ int reconstruct(char *name, struct discovered *found)
/* updated by the counts_tobuf below, different in each file */
}
- rewind(newindex);
- reconstruct_counts_tobuf(buf, &mailbox, &index_counts);
- n = fwrite(buf, 1, INDEX_HEADER_SIZE, newindex);
- if (n != INDEX_HEADER_SIZE) {
- r = IMAP_IOERROR;
- goto bail;
- }
- rewind(newexpunge);
- reconstruct_counts_tobuf(buf, &mailbox, &expunge_counts);
- n = fwrite(buf, 1, INDEX_HEADER_SIZE, newexpunge);
- if (n != INDEX_HEADER_SIZE) {
- r = IMAP_IOERROR;
- goto bail;
- }
+ if( warn_only ) {
+ if( mailbox.exists != index_counts.newexists ) {
+ printf( "Update index message count, old: %i new: %i\n",
+ mailbox.exists, index_counts.newexists );
+ status++;
+ }
+ if( expunge_exists != expunge_counts.newexists ) {
+ printf( "Update expunge message count, old: %i new: %i\n",
+ expunge_exists, expunge_counts.newexists );
+ status++;
+ }
+ } else {
+ rewind(newindex);
+ reconstruct_counts_tobuf(buf, &mailbox, &index_counts);
+ n = fwrite(buf, 1, INDEX_HEADER_SIZE, newindex);
+ if (n != INDEX_HEADER_SIZE) {
+ r = IMAP_IOERROR;
+ goto bail;
+ }
+ rewind(newexpunge);
+ reconstruct_counts_tobuf(buf, &mailbox, &expunge_counts);
+ n = fwrite(buf, 1, INDEX_HEADER_SIZE, newexpunge);
+ if (n != INDEX_HEADER_SIZE) {
+ r = IMAP_IOERROR;
+ goto bail;
+ }
- fflush(newindex);
- fflush(newexpunge);
- if (ferror(newindex) || ferror(newexpunge) || fsync(newcache_fd) ||
- fsync(fileno(newindex)) || fsync(fileno(newexpunge))) {
- r = IMAP_IOERROR;
- goto bail;
- }
+ fflush(newindex);
+ fflush(newexpunge);
+ if (ferror(newindex) || ferror(newexpunge) || fsync(newcache_fd) ||
+ fsync(fileno(newindex)) || fsync(fileno(newexpunge))) {
+ r = IMAP_IOERROR;
+ goto bail;
+ }
- /* Free temporary resources now that the index/expunge update is done */
- close(newcache_fd);
- fclose(newexpunge);
- fclose(newindex);
+ /* Free temporary resources now that the index/expunge update is done */
+ close(newcache_fd);
+ fclose(newexpunge);
+ fclose(newindex);
+ }
if (expunge_base) map_free(&expunge_base, &expunge_len);
if (expunge_fd >= 0) close(expunge_fd);
@@ -1249,16 +1315,24 @@ int reconstruct(char *name, struct discovered *found)
/* this may change uniqueid, but if it does, nothing we can do
about it */
- mailbox_make_uniqueid(mailbox.name, mailbox.uidvalidity, unique_buf,
+ if( warn_only ) {
+ printf( "No uid for mailbox: %s\n", mailbox.name );
+ } else {
+ mailbox_make_uniqueid(mailbox.name, mailbox.uidvalidity, unique_buf,
sizeof(unique_buf));
- mailbox.uniqueid = xstrdup(unique_buf);
+ mailbox.uniqueid = xstrdup(unique_buf);
+ }
} else {
if (find_uniqid (mailbox.name, mailbox.uniqueid) != NULL ) {
-
- mailbox_make_uniqueid(mailbox.name, mailbox.uidvalidity,
+ if( warn_only ) {
+ printf( "Mailbox uid not unique: %s\n", mailbox.name );
+ status++;
+ } else {
+ mailbox_make_uniqueid(mailbox.name, mailbox.uidvalidity,
unique_buf, sizeof(unique_buf));
- free (mailbox.uniqueid);
- mailbox.uniqueid = xstrdup(unique_buf);
+ free (mailbox.uniqueid);
+ mailbox.uniqueid = xstrdup(unique_buf);
+ }
}
}
if (add_uniqid (mailbox.name, mailbox.uniqueid) == NULL) {
@@ -1272,33 +1346,35 @@ int reconstruct(char *name, struct discovered *found)
* mailbox_write_header() locks the new header file before it commits.
* That lock is only released on mailbox_close().
*/
- r = mailbox_write_header(&mailbox);
- if (!r)
- r = reconstruct_rename_single(&mailbox,
+ if( !warn_only ) {
+ r = mailbox_write_header(&mailbox);
+ if (!r)
+ r = reconstruct_rename_single(&mailbox,
IMAP_ENUM_METAPARTITION_FILES_CACHE,
FNAME_CACHE);
- if (!r)
+ if (!r)
r = reconstruct_rename_single(&mailbox,
IMAP_ENUM_METAPARTITION_FILES_EXPUNGE,
FNAME_EXPUNGE_INDEX);
- if (expunge_counts.newexists == 0) {
- reconstruct_delete_single(&mailbox,
+ if (expunge_counts.newexists == 0) {
+ reconstruct_delete_single(&mailbox,
IMAP_ENUM_METAPARTITION_FILES_EXPUNGE,
FNAME_EXPUNGE_INDEX, NULL);
- }
- if (!r)
- r = reconstruct_rename_single(&mailbox,
+ }
+ if (!r)
+ r = reconstruct_rename_single(&mailbox,
IMAP_ENUM_METAPARTITION_FILES_INDEX,
FNAME_INDEX);
- if (r) {
- mailbox_close(&mailbox);
- return (r);
- }
+ if (r) {
+ mailbox_close(&mailbox);
+ return (r);
+ }
- r = seen_reconstruct(&mailbox,
+ r = seen_reconstruct(&mailbox,
(time_t)0, (time_t)0, (int (*)())0, (void *)0);
- if (syncflag) {
- sync_log_mailbox(mailbox.name);
+ if (syncflag) {
+ sync_log_mailbox(mailbox.name);
+ }
}
mailbox_close(&mailbox);
diff --git a/man/reconstruct.8 b/man/reconstruct.8
index a8c2edc..040c73d 100644
--- a/man/reconstruct.8
+++ b/man/reconstruct.8
@@ -64,6 +64,12 @@ reconstruct \- reconstruct mailboxes
]
.br
[
+.B \-w
+]
+[
+.B \-e
+]
+[
.B \-k
]
[
@@ -145,9 +151,23 @@ Examine the filesystem underneath mailbox, adding all directories with
a cyrus.header found there as new mailboxes. Useful for
restoring mailboxes from backups.
.TP
+.B \-w
+Warn about potentially user visible changes to meta data without modifying
+any files, e.g., If the message count listed in the index or expunge meta
+data will change, it displays a summary of what the results will be and
+exits with status 1. Other potentially major changes are warned about.
+Since the cache file isn't user visible, the warn option doesn't generate
+any output regarding it, but is not rebuilt when \-w is used.
+.TP
+.B \-e
+If there are messages on the filesystem that are not listed in the index,
+add them to the expunge data. Useful if the expunge file is corrupted
+and you don't want all the users expunge messages to re-appear in their
+mailbox.
+.TP
.B \-k
Preserve the cyrus.expunge file and the corresponding message files
-instead of deleting them.
+instead of deleting them. (Default if delayed_expunge mode is configured.)
.TP
.B \-s
Adds synchronization records to the log, so the synchronization
@@ -185,6 +205,12 @@ to run:
reconstruct -r user/ben.lacy
.fi
+If you want to reconstruct a mailbox and generate GUID data only if the visible
+messages won't change, you could run:
+
+.nf
+reconstruct -w user/foo && reconstruct -g user/foo
+
.SH FILES
.TP
.B /etc/imapd.conf
----
Cyrus Home Page: http://cyrusimap.web.cmu.edu/
Cyrus Wiki/FAQ: http://cyrusimap.web.cmu.edu/twiki
List Archives/Info: http://asg.web.cmu.edu/cyrus/mailing-list.html
