On Thu, Nov 11, 2010 at 01:45:34PM +1100, Bron Gondwana wrote:
> Now - I'm going to look into cyrus.expunge handling magic for you.
Here's the rest of that series - a fix against the first lot of
patches I sent which is necessary, otherwise replication breaks.
Oops. I did most testing of the rest of the world later...
And a fix against a pop3 issue you probably don't care too much
about right now ;)
And finally - a patch to reconstruct (even in -s mode) which will
look for a spare cyrus.expunge and just delete all the mentioned
records. It will let you put the cyrus.expunge files from backup
into the directories and then just clean up :) Hopefully. It
won't hurt to use it on folders which upgraded correctly either.
These patches are all on github:
http://github.com/brong/cyrus-imapd/
In the for244 branch on top of cyrus-imapd-2.4, and in the master
branch at CMU on top of master, and of course in the fastmail
branch which always lives on top of master.
Bron.
>From 732e6e1b30d6a67aa549a06093d6dc6929d9dfcb Mon Sep 17 00:00:00 2001
From: Bron Gondwana <brong@xxxxxxxxx>
Date: Thu, 11 Nov 2010 14:50:32 +1100
Subject: [PATCH 5/7] Bug 3335 - make pop3 expunge efficient
---
imap/pop3d.c | 61 ++++++++++++++++++++++++++++++++++++---------------------
1 files changed, 38 insertions(+), 23 deletions(-)
diff --git a/imap/pop3d.c b/imap/pop3d.c
index f96723f..e8ae12c 100644
--- a/imap/pop3d.c
+++ b/imap/pop3d.c
@@ -147,8 +147,6 @@ int popd_starttls_done = 0;
static int popd_myrights;
-static mailbox_decideproc_t expungedeleted;
-
/* the sasl proxy policy context */
static struct proxy_context popd_proxyctx = {
0, 1, &popd_authstate, NULL, NULL
@@ -813,6 +811,42 @@ void kpop(void)
}
#endif
+static int expunge_deleted(void)
+{
+ struct index_record record;
+ uint32_t msgno;
+ int r = 0;
+
+ /* loop over all known messages looking for deletes */
+ for (msgno = 1; msgno <= popd_exists; msgno++) {
+ /* not deleted? skip */
+ if (!popd_msg[msgno].deleted)
+ continue;
+
+ /* error reading? abort */
+ r = mailbox_read_index_record(popd_mailbox, popd_msg[msgno].recno, &record);
+ if (r) break;
+
+ /* already expunged? skip */
+ if (record.system_flags & FLAG_EXPUNGED)
+ continue;
+
+ /* mark expunged */
+ record.system_flags |= FLAG_EXPUNGED;
+
+ /* store back to the mailbox */
+ r = mailbox_rewrite_index_record(popd_mailbox, &record);
+ if (r) break;
+ }
+
+ if (r) {
+ syslog(LOG_ERR, "IOERROR: %s failed to expunge record %u uid %u, aborting",
+ popd_mailbox->name, msgno, popd_msg[msgno].uid);
+ }
+
+ return r;
+}
+
/*
* Top-level command loop parsing
*/
@@ -931,12 +965,8 @@ static void cmdloop(void)
if (msgno <= popd_exists)
update_seen();
- /* check for changes */
- for (msgno = 1; msgno <= popd_exists; msgno++)
- if (popd_msg[msgno].deleted) break;
-
- if (msgno <= popd_exists)
- mailbox_expunge(popd_mailbox, expungedeleted, NULL, NULL);
+ /* process looking for deleted messages */
+ expunge_deleted();
mailbox_commit(popd_mailbox);
mailbox_unlock_index(popd_mailbox, NULL);
@@ -1905,21 +1935,6 @@ static int blat(int msgno, int lines)
return 0;
}
-static unsigned expungedeleted(struct mailbox *mailbox __attribute__((unused)),
- struct index_record *record,
- void *rock __attribute__((unused)))
-{
- uint32_t msgno;
-
- /* XXX - could make this more efficient with binary search */
- for (msgno = 1; msgno <= popd_exists; msgno++) {
- if (popd_msg[msgno].uid == record->uid) {
- return popd_msg[msgno].deleted;
- }
- }
- return 0;
-}
-
/* Reset the given sasl_conn_t to a sane state */
static int reset_saslconn(sasl_conn_t **conn)
{
--
1.7.1
>From 9d52edfb1d24ca4b101570be6f9625038385c6fc Mon Sep 17 00:00:00 2001
From: Bron Gondwana <brong@xxxxxxxxx>
Date: Thu, 11 Nov 2010 15:26:27 +1100
Subject: [PATCH 6/7] Update counts fix - don't break sync_crc
---
imap/mailbox.c | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/imap/mailbox.c b/imap/mailbox.c
index 045261a..c904d97 100644
--- a/imap/mailbox.c
+++ b/imap/mailbox.c
@@ -1962,7 +1962,7 @@ int mailbox_index_recalc(struct mailbox *mailbox)
for (recno = 1; recno <= mailbox->i.num_records; recno++) {
r = mailbox_read_index_record(mailbox, recno, &record);
if (r) return r;
- header_update_counts(&mailbox->i, &record, 1);
+ mailbox_index_update_counts(mailbox, &record, 1);
}
return 0;
--
1.7.1
>From ee79061faf8e0dfce154a60c981bc6856865902d Mon Sep 17 00:00:00 2001
From: Bron Gondwana <brong@xxxxxxxxx>
Date: Thu, 11 Nov 2010 15:21:55 +1100
Subject: [PATCH 7/7] Use old cyrus.expunge file to clean out bogus records.
---
imap/mailbox.c | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 files changed, 72 insertions(+), 0 deletions(-)
diff --git a/imap/mailbox.c b/imap/mailbox.c
index c904d97..d7c7e9e 100644
--- a/imap/mailbox.c
+++ b/imap/mailbox.c
@@ -3227,6 +3227,75 @@ static int find_files(struct mailbox *mailbox, struct found_files *files,
return 0;
}
+static void cleanup_stale_expunged(struct mailbox *mailbox)
+{
+ const char *fname;
+ int expunge_fd = -1;
+ const char *expunge_base = NULL;
+ unsigned long expunge_len = 0; /* mapped size */
+ unsigned long expunge_num;
+ unsigned long emapnum;
+ uint32_t erecno;
+ uint32_t uid;
+ bit32 eoffset, expungerecord_size;
+ const char *bufp;
+ struct stat sbuf;
+ int count = 0;
+ int r;
+
+ /* it's always read-writes */
+ fname = mailbox_meta_fname(mailbox, META_EXPUNGE);
+ expunge_fd = open(fname, O_RDWR, 0);
+ if (expunge_fd == -1)
+ goto done; /* yay, no crappy expunge file */
+
+ /* boo - gotta read and find out the UIDs */
+ r = fstat(expunge_fd, &sbuf);
+ if (r == -1)
+ goto done;
+
+ if (sbuf.st_size < INDEX_HEADER_SIZE)
+ goto done;
+
+ map_refresh(expunge_fd, 1, &expunge_base,
+ &expunge_len, sbuf.st_size, "expunge",
+ mailbox->name);
+
+ /* use the expunge file's header information just in case
+ * versions are skewed for some reason */
+ eoffset = ntohl(*((bit32 *)(expunge_base+OFFSET_START_OFFSET)));
+ expungerecord_size = ntohl(*((bit32 *)(expunge_base+OFFSET_RECORD_SIZE)));
+
+ /* bogus data at the start of the expunge file? */
+ if (!eoffset || !expungerecord_size)
+ goto done;
+
+ expunge_num = ntohl(*((bit32 *)(expunge_base+OFFSET_NUM_RECORDS)));
+ emapnum = (sbuf.st_size - eoffset) / expungerecord_size;
+ if (emapnum < expunge_num) {
+ expunge_num = emapnum;
+ }
+
+ /* add every UID to the files list */
+ for (erecno = 1; erecno <= expunge_num; erecno++) {
+ bufp = expunge_base + eoffset + (erecno-1)*expungerecord_size;
+ uid = ntohl(*((bit32 *)(bufp+OFFSET_UID)));
+ fname = mailbox_message_fname(mailbox, uid);
+ unlink(fname);
+ count++;
+ }
+
+ printf("%s removed %d records from stale cyrus.expunge\n",
+ mailbox->name, count);
+
+ fname = mailbox_meta_fname(mailbox, META_EXPUNGE);
+ unlink(fname);
+
+done:
+ if (expunge_base) map_free(&expunge_base, &expunge_len);
+ if (expunge_fd != -1) close(expunge_fd);
+}
+
/* this is kind of like mailbox_create, but we try to rescue
* what we can from the filesystem! */
static int mailbox_reconstruct_create(const char *name, struct mailbox **mbptr)
@@ -3774,6 +3843,9 @@ int mailbox_reconstruct(const char *name, int flags)
mailbox->need_cache_refresh = 1;
}
+ /* find cyrus.expunge file if present */
+ cleanup_stale_expunged(mailbox);
+
r = find_files(mailbox, &files, flags);
if (r) goto close;
init_files(&discovered);
--
1.7.1
----
Cyrus Home Page: http://www.cyrusimap.org/
List Archives/Info: http://lists.andrew.cmu.edu/pipermail/info-cyrus/
