On Wed, Nov 10, 2010 at 09:38:34PM +0100, Paul Dekkers wrote:
> Hi Bron,
>
> Thanks for your reply:
>
> On 10-11-2010 21:27, Bron Gondwana wrote:
>
> >>(Can I downgrade? I just need to reconstruct everything, right, because
> >>the index format changed? :-S)
> >
> >Yes, you will need to reconstruct anything which successfully upgraded.
>
> Ok, there's no way of telling what folders got succesfully upgraded,
> right? I only see messages "Index upgrade failed: user.paul",
> indicating the ones that failed :-S
>
> (A selective reconstruct would be faster.)
You can run 'reconstruct -s', which is pretty fast, but still
detects the worst cases.
> >This issue got reported last night, and I wrote a patch but haven't
>
> So, just the reconstruct problem, or more of my symptoms? (I've
> looked at the devel archive and the info-cyrus, but did not see
> anything similar, or from last night anyway ;-) so I have no idea
> what issue exactly.)
This will fix all your broken upgrades.
> >yet tested it. I can give it to you in about 2-3 hours if you'll still
> >be around then!
>
> Hmm, 3 hours will be quite late here, but if you can help me
> understand if the patch would solve "all" my symptoms, it might be
> worth it.
>
> But I have the feeling its not just reconstruct (as even after
> succesfull reconstructs I still get errors, and I didn't expect this
> many errors initially anyway).
No - upgrades were all sorts of broken :( I'm really sorry. This
patch will get scrambled for 2.4.4.
(Note: I havne't written the cyrus.expunge fix yet - this is just to
make the rest of the upgrades go cleanly!)
https://github.com/brong/cyrus-imapd/tree/for244
Or just apply the attached patches (generated from that tree since
2.4.3 was released)
Patch number 4 is the one you care about, but the others are fine
to apply too.
Now - I'm going to look into cyrus.expunge handling magic for you.
Bron.
>From 7380370a4825071ce460648c4c49a3aa9603f13b Mon Sep 17 00:00:00 2001
From: Bron Gondwana <brong@xxxxxxxxx>
Date: Tue, 9 Nov 2010 16:28:59 +1100
Subject: [PATCH 1/4] Bug 3331 - sync_reset broken
---
imap/sync_reset.c | 10 ++++++++--
1 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/imap/sync_reset.c b/imap/sync_reset.c
index 4c81efb..fbb0637 100644
--- a/imap/sync_reset.c
+++ b/imap/sync_reset.c
@@ -86,7 +86,7 @@
/* global state */
const int config_need_data = 0;
-/* Static global variables and support routines for sync_server */
+/* Static global variables and support routines for sync_reset */
extern char *optarg;
extern int optind;
@@ -103,6 +103,9 @@ static void shut_down(int code)
{
in_shutdown = 1;
+ annotatemore_close();
+ annotatemore_done();
+
if (sync_userid) free(sync_userid);
if (sync_authstate) auth_freestate(sync_authstate);
@@ -126,7 +129,7 @@ static int usage(const char *name)
void fatal(const char* s, int code)
{
- fprintf(stderr, "sync_server: %s\n", s);
+ fprintf(stderr, "sync_reset: %s\n", s);
exit(code);
}
@@ -241,6 +244,9 @@ main(int argc, char **argv)
signals_set_shutdown(&shut_down);
signals_add_handlers(0);
+ annotatemore_init(0, NULL, NULL);
+ annotatemore_open(NULL);
+
if (!force) {
fprintf(stderr, "Usage: sync_reset -f user user user ...\n");
fprintf(stderr, " -f [force] is obligitory for safety\n");
--
1.7.1
>From 86cc6436f18eaef7df4c55d8368c890afdc64cd9 Mon Sep 17 00:00:00 2001
From: Bron Gondwana <brong@xxxxxxxxx>
Date: Wed, 10 Nov 2010 08:36:26 +1100
Subject: [PATCH 2/4] Bug 3332 - fix subscriptions handling during unuser/reset
---
imap/sync_client.c | 2 +-
imap/sync_reset.c | 2 +-
imap/sync_server.c | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/imap/sync_client.c b/imap/sync_client.c
index 074fd9b..283ce12 100644
--- a/imap/sync_client.c
+++ b/imap/sync_client.c
@@ -1764,7 +1764,7 @@ int do_user_sub(const char *userid, struct sync_name_list *replica_subs)
for (msubs = master_subs->head; msubs; msubs = msubs->next) {
r = (sync_namespace.mboxname_tointernal)(&sync_namespace, msubs->name,
userid, buf);
- if (r) continue;
+ if (r) continue; /* just ignore invalid subs */
rsubs = sync_name_lookup(replica_subs, buf);
if (rsubs) {
rsubs->mark = 1;
diff --git a/imap/sync_reset.c b/imap/sync_reset.c
index fbb0637..db30134 100644
--- a/imap/sync_reset.c
+++ b/imap/sync_reset.c
@@ -149,11 +149,11 @@ static int reset_single(const char *userid)
addmbox_sub, (void *)list, 1);
if (r) goto fail;
+ /* ignore failures here - the subs file gets deleted soon anyway */
for (item = list->head; item; item = item->next) {
r = (sync_namespacep->mboxname_tointernal)(sync_namespacep, item->name,
userid, buf);
if (!r) r = mboxlist_changesub(buf, userid, sync_authstate, 0, 0);
- if (r) goto fail;
}
sync_name_list_free(&list);
diff --git a/imap/sync_server.c b/imap/sync_server.c
index e87867e..2218a04 100644
--- a/imap/sync_server.c
+++ b/imap/sync_server.c
@@ -1838,11 +1838,11 @@ static int do_unuser(struct dlist *kin)
addmbox_sub, (void *)list, 1);
if (r) goto fail;
+ /* ignore failures here - the subs file gets deleted soon anyway */
for (item = list->head; item; item = item->next) {
r = (sync_namespacep->mboxname_tointernal)(sync_namespacep, item->name,
userid, buf);
if (!r) r = mboxlist_changesub(buf, userid, sync_authstate, 0, 0);
- if (r) goto fail;
}
sync_name_list_free(&list);
--
1.7.1
>From 20499abe02a62bd36eaf4fe4d24e3622230a0fa5 Mon Sep 17 00:00:00 2001
From: Bron Gondwana <brong@xxxxxxxxx>
Date: Wed, 10 Nov 2010 11:31:05 +1100
Subject: [PATCH 3/4] Bug 3333 - fix proc_registration in imapd, and refactor login success
---
imap/fud.c | 12 +++----
imap/imapd.c | 97 ++++++++++++++++++++++++---------------------------
imap/nntpd.c | 9 +----
imap/pop3d.c | 14 ++-----
imap/proc.c | 13 ++-----
imap/proc.h | 56 ++++++++++++++++++++++++++++++
imap/smmapd.c | 6 ++--
imap/sync_server.c | 8 +---
8 files changed, 122 insertions(+), 93 deletions(-)
create mode 100644 imap/proc.h
diff --git a/imap/fud.c b/imap/fud.c
index 27c08d9..34d8761 100644
--- a/imap/fud.c
+++ b/imap/fud.c
@@ -63,19 +63,20 @@
#include <errno.h>
#include <pwd.h>
+#include "acl.h"
#include "assert.h"
#include "mboxlist.h"
#include "global.h"
#include "exitcodes.h"
#include "imap_err.h"
#include "mailbox.h"
+#include "map.h"
+#include "mboxname.h"
+#include "proc.h"
+#include "seen.h"
#include "xmalloc.h"
#include "xstrlcpy.h"
#include "xstrlcat.h"
-#include "acl.h"
-#include "seen.h"
-#include "mboxname.h"
-#include "map.h"
#define REQ_OK 0
#define REQ_DENY 1
@@ -90,9 +91,6 @@ static struct namespace fud_namespace;
* only if we're not on a frontend, so we won't flat-out require it here */
const int config_need_data = 0;
-/* forward decls */
-extern void setproctitle_init(int argc, char **argv, char **envp);
-
int handle_request(const char *who, const char *name,
struct sockaddr_storage sfrom, socklen_t sfromsiz);
diff --git a/imap/imapd.c b/imap/imapd.c
index d03bd14..b8b0731 100644
--- a/imap/imapd.c
+++ b/imap/imapd.c
@@ -96,6 +96,7 @@
#include "mbdump.h"
#include "mkgmtime.h"
#include "mupdate-client.h"
+#include "proc.h"
#include "quota.h"
#include "seen.h"
#include "statuscache.h"
@@ -379,11 +380,6 @@ static int subscribed_cb(char *name, int matchlen, int maycreate,
static void list_data(struct listargs *listargs);
static void list_data_remote(char *tag, struct listargs *listargs);
-extern void setproctitle_init(int argc, char **argv, char **envp);
-extern int proc_register(const char *progname, const char *clienthost,
- const char *userid, const char *mailbox);
-extern void proc_cleanup(void);
-
extern int saslserver(sasl_conn_t *conn, const char *mech,
const char *init_resp, const char *resp_prefix,
const char *continuation, const char *empty_resp,
@@ -2078,6 +2074,36 @@ void cmdloop()
}
}
+static void authentication_success(void)
+{
+ int r;
+
+ /* register the user */
+ proc_register("imapd", imapd_clienthost, imapd_userid, NULL);
+
+ /* authstate already created by mysasl_proxy_policy() */
+ imapd_userisadmin = global_authisa(imapd_authstate, IMAPOPT_ADMINS);
+
+ /* Create telemetry log */
+ imapd_logfd = telemetry_log(imapd_userid, imapd_in, imapd_out, 0);
+
+ /* Set namespace */
+ r = mboxname_init_namespace(&imapd_namespace,
+ imapd_userisadmin || imapd_userisproxyadmin);
+ if (r) {
+ syslog(LOG_ERR, "%s", error_message(r));
+ fatal(error_message(r), EC_CONFIG);
+ }
+
+ /* Make a copy of the external userid for use in proxying */
+ proxy_userid = xstrdup(imapd_userid);
+
+ /* Translate any separators in userid */
+ mboxname_hiersep_tointernal(&imapd_namespace, imapd_userid,
+ config_virtdomains ?
+ strcspn(imapd_userid, "@") : 0);
+}
+
/*
* Perform a LOGIN command
*/
@@ -2230,41 +2256,20 @@ void cmd_login(char *tag, char *user)
plaintextloginalert = config_getstring(IMAPOPT_PLAINTEXTLOGINALERT);
}
}
-
- /* authstate already created by mysasl_proxy_policy() */
- imapd_userisadmin = global_authisa(imapd_authstate, IMAPOPT_ADMINS);
+
+ buf_free(&passwdbuf);
prot_printf(imapd_out, "%s OK [CAPABILITY ", tag);
capa_response(CAPA_PREAUTH|CAPA_POSTAUTH);
prot_printf(imapd_out, "] %s\r\n", reply);
- /* Create telemetry log */
- imapd_logfd = telemetry_log(imapd_userid, imapd_in, imapd_out, 0);
-
- /* Set namespace */
- if ((r = mboxname_init_namespace(&imapd_namespace,
- imapd_userisadmin || imapd_userisproxyadmin)) != 0) {
- syslog(LOG_ERR, "%s", error_message(r));
- fatal(error_message(r), EC_CONFIG);
- }
-
- /* Make a copy of the external userid for use in proxying */
- proxy_userid = xstrdup(imapd_userid);
-
- /* Translate any separators in userid */
- mboxname_hiersep_tointernal(&imapd_namespace, imapd_userid,
- config_virtdomains ?
- strcspn(imapd_userid, "@") : 0);
-
- buf_free(&passwdbuf);
- return;
+ authentication_success();
}
/*
* Perform an AUTHENTICATE command
*/
-void
-cmd_authenticate(char *tag, char *authtype, char *resp)
+void cmd_authenticate(char *tag, char *authtype, char *resp)
{
int sasl_result;
@@ -2356,8 +2361,6 @@ cmd_authenticate(char *tag, char *authtype, char *resp)
imapd_userid = xstrdup(canon_user);
}
- proc_register("imapd", imapd_clienthost, imapd_userid, (char *)0);
-
syslog(LOG_NOTICE, "login: %s %s%s %s%s %s", imapd_clienthost,
imapd_userid, imapd_magicplus ? imapd_magicplus : "",
authtype, imapd_starttls_done ? "+TLS" : "", "User logged in");
@@ -2396,25 +2399,7 @@ cmd_authenticate(char *tag, char *authtype, char *resp)
prot_setsasl(imapd_in, imapd_saslconn);
prot_setsasl(imapd_out, imapd_saslconn);
- /* Create telemetry log */
- imapd_logfd = telemetry_log(imapd_userid, imapd_in, imapd_out, 0);
-
- /* Set namespace */
- if ((r = mboxname_init_namespace(&imapd_namespace,
- imapd_userisadmin || imapd_userisproxyadmin)) != 0) {
- syslog(LOG_ERR, "%s", error_message(r));
- fatal(error_message(r), EC_CONFIG);
- }
-
- /* Make a copy of the external userid for use in proxying */
- proxy_userid = xstrdup(imapd_userid);
-
- /* Translate any separators in userid */
- mboxname_hiersep_tointernal(&imapd_namespace, imapd_userid,
- config_virtdomains ?
- strcspn(imapd_userid, "@") : 0);
-
- return;
+ authentication_success();
}
/*
@@ -3527,6 +3512,13 @@ void cmd_select(char *tag, char *cmd, char *name)
wasopen = 1;
}
+ if (wasopen) {
+ /* un-register currently selected mailbox, it may get
+ * overwritten later, but easier here than handling
+ * all possible error paths */
+ proc_register("imapd", imapd_clienthost, imapd_userid, NULL);
+ }
+
r = (*imapd_namespace.mboxname_tointernal)(&imapd_namespace, name,
imapd_userid, mailboxname);
@@ -3712,6 +3704,9 @@ void cmd_select(char *tag, char *cmd, char *name)
*/
void cmd_close(char *tag, char *cmd)
{
+ /* unregister the selected mailbox */
+ proc_register("imapd", imapd_clienthost, imapd_userid, NULL);
+
if (backend_current) {
/* remote mailbox */
prot_printf(backend_current->out, "%s %s\r\n", tag, cmd);
diff --git a/imap/nntpd.c b/imap/nntpd.c
index 79e3b80..270127b 100644
--- a/imap/nntpd.c
+++ b/imap/nntpd.c
@@ -97,6 +97,7 @@
#include "mkgmtime.h"
#include "mupdate-client.h"
#include "nntp_err.h"
+#include "proc.h"
#include "prot.h"
#include "proxy.h"
#include "retry.h"
@@ -234,12 +235,6 @@ static void cmd_starttls(int nntps);
void usage(void);
void shut_down(int code) __attribute__ ((noreturn));
-
-extern void setproctitle_init(int argc, char **argv, char **envp);
-extern int proc_register(const char *progname, const char *clienthost,
- const char *userid, const char *mailbox);
-extern void proc_cleanup(void);
-
extern int saslserver(sasl_conn_t *conn, const char *mech,
const char *init_resp, const char *resp_prefix,
const char *continuation, const char *empty_resp,
@@ -2197,7 +2192,7 @@ static void cmd_authinfo_sasl(char *cmd, char *mech, char *resp)
}
nntp_userid = xstrdup((const char *) val);
- proc_register("nntpd", nntp_clienthost, nntp_userid, (char *)0);
+ proc_register("nntpd", nntp_clienthost, nntp_userid, NULL);
syslog(LOG_NOTICE, "login: %s %s %s%s %s", nntp_clienthost, nntp_userid,
mech, nntp_starttls_done ? "+TLS" : "", "User logged in");
diff --git a/imap/pop3d.c b/imap/pop3d.c
index 4b0e115..f96723f 100644
--- a/imap/pop3d.c
+++ b/imap/pop3d.c
@@ -86,6 +86,7 @@
#include "idle.h"
#include "telemetry.h"
#include "backend.h"
+#include "proc.h"
#include "proxy.h"
#include "seen.h"
#include "userdeny.h"
@@ -198,12 +199,6 @@ static int update_seen(void);
void usage(void);
void shut_down(int code) __attribute__ ((noreturn));
-
-extern void setproctitle_init(int argc, char **argv, char **envp);
-extern int proc_register(const char *progname, const char *clienthost,
- const char *userid, const char *mailbox);
-extern void proc_cleanup(void);
-
extern int saslserver(sasl_conn_t *conn, const char *mech,
const char *init_resp, const char *resp_prefix,
const char *continuation, const char *empty_chal,
@@ -1751,8 +1746,6 @@ int openinbox(void)
!sdata.messages) {
/* local mailbox (empty) -- don't bother opening the mailbox */
syslog(LOG_INFO, "optimized mode for empty maildrop: %s", popd_userid);
-
- proc_register("pop3d", popd_clienthost, popd_userid, inboxname);
}
else {
/* local mailbox */
@@ -1827,8 +1820,6 @@ int openinbox(void)
}
popd_exists = msgno;
- proc_register("pop3d", popd_clienthost, popd_userid,
- popd_mailbox->name);
/* finished our initial read */
mailbox_unlock_index(popd_mailbox, NULL);
@@ -1846,6 +1837,9 @@ int openinbox(void)
}
}
+ /* register process */
+ proc_register("pop3d", popd_clienthost, popd_userid, inboxname);
+
/* Create telemetry log */
popd_logfd = telemetry_log(popd_userid, popd_in, popd_out, 0);
diff --git a/imap/proc.c b/imap/proc.c
index 11b3e6d..b1732e3 100644
--- a/imap/proc.c
+++ b/imap/proc.c
@@ -52,8 +52,9 @@
#include <syslog.h>
#include <string.h>
-#include "global.h"
#include "exitcodes.h"
+#include "global.h"
+#include "proc.h"
#include "xmalloc.h"
#define FNAME_PROCDIR "/proc/"
@@ -61,14 +62,8 @@
static char *procfname = 0;
static FILE *procfile = 0;
-extern void setproctitle_init(int argc, char **argv, char **envp);
-extern void setproctitle(const char *fmt, ...);
-
-int proc_register(progname, clienthost, userid, mailbox)
-const char *progname;
-const char *clienthost;
-const char *userid;
-const char *mailbox;
+int proc_register(const char *progname, const char *clienthost,
+ const char *userid, const char *mailbox)
{
unsigned pid;
int pos;
diff --git a/imap/proc.h b/imap/proc.h
new file mode 100644
index 0000000..c692c3d
--- /dev/null
+++ b/imap/proc.h
@@ -0,0 +1,56 @@
+/* proc.c -- Server process registry
+ *
+ * Copyright (c) 1994-2008 Carnegie Mellon University. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * 3. The name "Carnegie Mellon University" must not be used to
+ * endorse or promote products derived from this software without
+ * prior written permission. For permission or any legal
+ * details, please contact
+ * Carnegie Mellon University
+ * Center for Technology Transfer and Enterprise Creation
+ * 4615 Forbes Avenue
+ * Suite 302
+ * Pittsburgh, PA 15213
+ * (412) 268-7393, fax: (412) 268-7395
+ * innovation@xxxxxxxxxxxxxx
+ *
+ * 4. Redistributions of any form whatsoever must retain the following
+ * acknowledgment:
+ * "This product includes software developed by Computing Services
+ * at Carnegie Mellon University (http://www.cmu.edu/computing/)."
+ *
+ * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
+ * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
+ * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * $Id: proc.c,v 1.27 2010/01/06 17:01:38 murch Exp $
+ */
+
+#ifndef _PROC_H
+#define _PROC_H
+
+extern void setproctitle_init(int argc, char **argv, char **envp);
+extern void setproctitle(const char *fmt, ...);
+
+extern int proc_register(const char *progname, const char *clienthost,
+ const char *userid, const char *mailbox);
+
+extern void proc_cleanup(void);
+
+#endif /* _PROC_H */
diff --git a/imap/smmapd.c b/imap/smmapd.c
index ddb9699..90d56de 100644
--- a/imap/smmapd.c
+++ b/imap/smmapd.c
@@ -89,11 +89,12 @@
#include "acl.h"
#include "append.h"
-#include "mboxlist.h"
-#include "global.h"
#include "exitcodes.h"
+#include "global.h"
#include "imap_err.h"
+#include "mboxlist.h"
#include "mupdate-client.h"
+#include "proc.h"
#include "util.h"
#include "xmalloc.h"
#include "xstrlcpy.h"
@@ -113,7 +114,6 @@ static struct namespace map_namespace;
const int config_need_data = 0;
/* forward decls */
-extern void setproctitle_init(int argc, char **argv, char **envp);
int begin_handling(void);
void smmapd_reset(void)
diff --git a/imap/sync_server.c b/imap/sync_server.c
index 2218a04..47c70f2 100644
--- a/imap/sync_server.c
+++ b/imap/sync_server.c
@@ -86,6 +86,7 @@
#include "mailbox.h"
#include "map.h"
#include "mboxlist.h"
+#include "proc.h"
#include "prot.h"
#include "quota.h"
#include "retry.h"
@@ -151,11 +152,6 @@ static void cmd_apply(struct dlist *kl,
void usage(void);
void shut_down(int code) __attribute__ ((noreturn));
-extern void setproctitle_init(int argc, char **argv, char **envp);
-extern int proc_register(const char *progname, const char *clienthost,
- const char *userid, const char *mailbox);
-extern void proc_cleanup(void);
-
extern int saslserver(sasl_conn_t *conn, const char *mech,
const char *init_resp, const char *resp_prefix,
const char *continuation, const char *empty_resp,
@@ -820,7 +816,7 @@ static void cmd_authenticate(char *mech, char *resp)
}
sync_userid = xstrdup((const char *) val);
- proc_register("sync_server", sync_clienthost, sync_userid, (char *)0);
+ proc_register("sync_server", sync_clienthost, sync_userid, NULL);
syslog(LOG_NOTICE, "login: %s %s %s%s %s", sync_clienthost, sync_userid,
mech, sync_starttls_done ? "+TLS" : "", "User logged in");
--
1.7.1
>From 21bf6a722d3bd88cd79d6ed54f3a6fba2842a438 Mon Sep 17 00:00:00 2001
From: Bron Gondwana <brong@xxxxxxxxxxxxxxxxxxxx>
Date: Wed, 10 Nov 2010 21:35:07 +1100
Subject: [PATCH 4/4] Bug 3334 - simplify upgrade path to be much more robust
---
imap/mailbox.c | 270 +++++++++++++++++++++++++++++++++----------------
imap/mailbox.h | 26 +++--
imap/upgrade_index.c | 266 ++++++++++++++-----------------------------------
3 files changed, 274 insertions(+), 288 deletions(-)
diff --git a/imap/mailbox.c b/imap/mailbox.c
index a636085..045261a 100644
--- a/imap/mailbox.c
+++ b/imap/mailbox.c
@@ -1242,7 +1242,7 @@ int mailbox_user_flag(struct mailbox *mailbox, const char *flag,
return 0;
}
-int mailbox_buf_to_index_header(struct index_header *i, const char *buf)
+int mailbox_buf_to_index_header(const char *buf, struct index_header *i)
{
uint32_t crc;
@@ -1345,7 +1345,7 @@ static int mailbox_read_index_header(struct mailbox *mailbox)
if (mailbox->index_size < INDEX_HEADER_SIZE)
return IMAP_MAILBOX_BADFORMAT;
- r = mailbox_buf_to_index_header(&mailbox->i, mailbox->index_base);
+ r = mailbox_buf_to_index_header(mailbox->index_base, &mailbox->i);
if (r) return r;
r = mailbox_refresh_index_map(mailbox);
@@ -1891,7 +1891,7 @@ static void mailbox_quota_dirty(struct mailbox *mailbox)
}
}
-void mailbox_index_update_counts(struct mailbox *mailbox,
+static void header_update_counts(struct index_header *i,
struct index_record *record,
int is_add)
{
@@ -1901,35 +1901,43 @@ void mailbox_index_update_counts(struct mailbox *mailbox,
if (record->system_flags & FLAG_EXPUNGED)
return;
- mailbox_quota_dirty(mailbox);
-
/* update mailbox header fields */
if (record->system_flags & FLAG_ANSWERED)
- mailbox->i.answered += num;
+ i->answered += num;
if (record->system_flags & FLAG_FLAGGED)
- mailbox->i.flagged += num;
+ i->flagged += num;
if (record->system_flags & FLAG_DELETED)
- mailbox->i.deleted += num;
+ i->deleted += num;
if (is_add) {
- mailbox->i.exists++;
- mailbox->i.quota_mailbox_used += record->size;
+ i->exists++;
+ i->quota_mailbox_used += record->size;
}
else {
- if (mailbox->i.exists)
- mailbox->i.exists--;
+ if (i->exists) i->exists--;
/* corruption prevention - check we don't go negative */
- if (mailbox->i.quota_mailbox_used > record->size)
- mailbox->i.quota_mailbox_used -= record->size;
+ if (i->quota_mailbox_used > record->size)
+ i->quota_mailbox_used -= record->size;
else
- mailbox->i.quota_mailbox_used = 0;
+ i->quota_mailbox_used = 0;
}
- mailbox->i.sync_crc ^= make_sync_crc(mailbox, record);
+}
+static void mailbox_index_update_counts(struct mailbox *mailbox,
+ struct index_record *record,
+ int is_add)
+{
+ /* avoid dirtying for expunged */
+ if (record->system_flags & FLAG_EXPUNGED)
+ return;
+ mailbox_quota_dirty(mailbox);
mailbox_index_dirty(mailbox);
+ header_update_counts(&mailbox->i, record, is_add);
+ /* xor doesn't care if it's an add! */
+ mailbox->i.sync_crc ^= make_sync_crc(mailbox, record);
}
int mailbox_index_recalc(struct mailbox *mailbox)
@@ -1954,7 +1962,7 @@ int mailbox_index_recalc(struct mailbox *mailbox)
for (recno = 1; recno <= mailbox->i.num_records; recno++) {
r = mailbox_read_index_record(mailbox, recno, &record);
if (r) return r;
- mailbox_index_update_counts(mailbox, &record, 1);
+ header_update_counts(&mailbox->i, &record, 1);
}
return 0;
@@ -2023,6 +2031,7 @@ int mailbox_rewrite_index_record(struct mailbox *mailbox,
/* remove the counts for the old copy, and add them for
* the new copy */
+
mailbox_index_update_counts(mailbox, &oldrecord, 0);
mailbox_index_update_counts(mailbox, record, 1);
@@ -2207,36 +2216,166 @@ static int mailbox_index_unlink(struct mailbox *mailbox)
return 0;
}
-/* need a mailbox exclusive lock, we're rewriting files */
-static int mailbox_index_repack(struct mailbox *mailbox)
+int mailbox_repack_setup(struct mailbox *mailbox,
+ struct mailbox_repack **repackptr)
{
- char *fname;
- uint32_t newrecno = 1;
+ struct mailbox_repack *repack = xzmalloc(sizeof(struct mailbox_repack));
+ const char *fname;
indexbuffer_t ibuf;
unsigned char *buf = ibuf.buf;
- uint32_t recno;
- int newindex_fd = -1, newcache_fd = -1;
- struct index_record record;
- size_t offset;
- int newgeneration;
- int r = IMAP_IOERROR;
int n;
- syslog(LOG_INFO, "Repacking mailbox %s", mailbox->name);
+ /* init */
+ repack->mailbox = mailbox;
+ repack->i = mailbox->i; /* struct copy */
+ repack->newindex_fd = -1;
+ repack->newcache_fd = -1;
+ /* new files */
fname = mailbox_meta_newfname(mailbox, META_INDEX);
- newindex_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666);
- if (newindex_fd == -1) goto fail;
+ repack->newindex_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666);
+ if (repack->newindex_fd == -1) goto fail;
fname = mailbox_meta_newfname(mailbox, META_CACHE);
- newcache_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666);
- if (newcache_fd == -1) goto fail;
+ repack->newcache_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666);
+ if (repack->newcache_fd == -1) goto fail;
/* update the generation number */
- newgeneration = mailbox->i.generation_no + 1;
- *((bit32 *)(buf)) = htonl(newgeneration);
- n = retry_write(newcache_fd, buf, 4);
- if (n != 4) goto fail;
+ repack->i.generation_no++;
+
+ /* zero out some values */
+ repack->i.num_records = 0;
+ repack->i.quota_mailbox_used = 0;
+ repack->i.num_records = 0;
+ repack->i.sync_crc = 0; /* no records is blank */
+ repack->i.answered = 0;
+ repack->i.deleted = 0;
+ repack->i.flagged = 0;
+ repack->i.exists = 0;
+ repack->i.first_expunged = 0;
+ repack->i.leaked_cache_records = 0;
+
+ /* prepare initial header buffer */
+ mailbox_index_header_to_buf(&repack->i, buf);
+
+ /* write initial headers */
+ n = retry_write(repack->newcache_fd, buf, 4);
+ if (n == -1) goto fail;
+
+ n = retry_write(repack->newindex_fd, buf, INDEX_HEADER_SIZE);
+ if (n == -1) goto fail;
+
+ *repackptr = repack;
+ return 0;
+
+ fail:
+ mailbox_repack_abort(&repack);
+ return IMAP_IOERROR;
+}
+
+int mailbox_repack_add(struct mailbox_repack *repack,
+ struct index_record *record)
+{
+ int r;
+ int n;
+ indexbuffer_t ibuf;
+ unsigned char *buf = ibuf.buf;
+
+ /* write out the new cache record - need to clear the cache_offset
+ * so it gets reset in the new record */
+ record->cache_offset = 0;
+ r = cache_append_record(repack->newcache_fd, record);
+ if (r) return r;
+
+ /* update counters */
+ header_update_counts(&repack->i, record, 1);
+
+ /* write the index record out */
+ mailbox_index_record_to_buf(record, buf);
+ n = retry_write(repack->newindex_fd, buf, INDEX_RECORD_SIZE);
+ if (n == -1)
+ return IMAP_IOERROR;
+
+ repack->i.num_records++;
+
+ return 0;
+}
+
+/* clean up memory structures and abort repack */
+void mailbox_repack_abort(struct mailbox_repack **repackptr)
+{
+ struct mailbox_repack *repack = *repackptr;
+ if (!repack) return; /* safe against double-free */
+ if (repack->newcache_fd != -1) close(repack->newcache_fd);
+ unlink(mailbox_meta_newfname(repack->mailbox, META_CACHE));
+ if (repack->newindex_fd != -1) close(repack->newindex_fd);
+ unlink(mailbox_meta_newfname(repack->mailbox, META_INDEX));
+ free(repack);
+ *repackptr = NULL;
+}
+
+int mailbox_repack_commit(struct mailbox_repack **repackptr)
+{
+ indexbuffer_t ibuf;
+ unsigned char *buf = ibuf.buf;
+ struct mailbox_repack *repack = *repackptr;
+ int n;
+ int r;
+
+ assert(repack);
+
+ repack->i.last_repack_time = time(0);
+
+ /* rewrite the header with updated details */
+ mailbox_index_header_to_buf(&repack->i, buf);
+ n = lseek(repack->newindex_fd, 0, SEEK_SET);
+ if (n == -1) {
+ r = IMAP_IOERROR;
+ goto fail;
+ }
+ n = retry_write(repack->newindex_fd, buf, INDEX_HEADER_SIZE);
+ if (n == -1) {
+ r = IMAP_IOERROR;
+ goto fail;
+ }
+
+ /* ensure everything is committed to disk */
+ if (fsync(repack->newindex_fd) || fsync(repack->newcache_fd))
+ goto fail;
+
+ close(repack->newcache_fd);
+ repack->newcache_fd = -1;
+ close(repack->newindex_fd);
+ repack->newindex_fd = -1;
+
+ /* rename index first - loader will handle un-renamed cache if
+ * the generation is lower */
+ r = mailbox_meta_rename(repack->mailbox, META_INDEX);
+ if (r) goto fail;
+
+ mailbox_meta_rename(repack->mailbox, META_CACHE);
+
+ free(repack);
+ *repackptr = NULL;
+ return 0;
+
+ fail:
+ mailbox_repack_abort(repackptr);
+ return r;
+}
+
+/* need a mailbox exclusive lock, we're rewriting files */
+static int mailbox_index_repack(struct mailbox *mailbox)
+{
+ struct mailbox_repack *repack = NULL;
+ uint32_t recno;
+ struct index_record record;
+ int r = IMAP_IOERROR;
+
+ syslog(LOG_INFO, "Repacking mailbox %s", mailbox->name);
+
+ r = mailbox_repack_setup(mailbox, &repack);
+ if (r) goto fail;
for (recno = 1; recno <= mailbox->i.num_records; recno++) {
r = mailbox_read_index_record(mailbox, recno, &record);
@@ -2247,8 +2386,8 @@ static int mailbox_index_repack(struct mailbox *mailbox)
/* just in case it was left lying around */
/* XXX - log error if unlink fails */
mailbox_message_unlink(mailbox, record.uid);
- if (record.modseq > mailbox->i.deletedmodseq)
- mailbox->i.deletedmodseq = record.modseq;
+ if (record.modseq > repack->i.deletedmodseq)
+ repack->i.deletedmodseq = record.modseq;
continue;
}
@@ -2256,63 +2395,18 @@ static int mailbox_index_repack(struct mailbox *mailbox)
r = mailbox_cacherecord(mailbox, &record);
if (r) goto fail;
- /* write out the new cache record - need to clear the cache_offset
- * so it gets reset in the new record */
- record.cache_offset = 0;
- r = cache_append_record(newcache_fd, &record);
+ r = mailbox_repack_add(repack, &record);
if (r) goto fail;
-
- offset = mailbox->i.start_offset +
- (newrecno-1) * mailbox->i.record_size;
-
- /* write the index record out */
- mailbox_index_record_to_buf(&record, buf);
- n = lseek(newindex_fd, offset, SEEK_SET);
- if (n == -1) {
- syslog(LOG_ERR, "IOERROR: seeking index record %u for %s: %m",
- recno, mailbox->name);
- goto fail;
- }
- n = retry_write(newindex_fd, buf, INDEX_RECORD_SIZE);
- if (n != INDEX_RECORD_SIZE) {
- syslog(LOG_ERR, "IOERROR: writing index record %u for %s: %m",
- recno, mailbox->name);
- goto fail;
- }
-
- newrecno++;
}
- /* update final header fields */
- mailbox->i.generation_no = newgeneration;
- mailbox->i.first_expunged = 0;
- mailbox->i.last_repack_time = time(0);
- mailbox->i.num_records = newrecno - 1;
- mailbox->i.leaked_cache_records = 0;
/* we unlinked any "needs unlink" in the process */
- mailbox->i.options &= ~(OPT_MAILBOX_NEEDS_REPACK|OPT_MAILBOX_NEEDS_UNLINK);
-
- mailbox_index_header_to_buf(&mailbox->i, buf);
- n = lseek(newindex_fd, 0, SEEK_SET);
- if (n == -1) goto fail;
- n = retry_write(newindex_fd, buf, INDEX_HEADER_SIZE);
- if (n != INDEX_HEADER_SIZE) goto fail;
-
- if (fsync(newindex_fd) || fsync(newcache_fd))
- goto fail;
-
- close(newcache_fd);
- close(newindex_fd);
-
- r = mailbox_meta_rename(mailbox, META_INDEX);
- if (!r) r = mailbox_meta_rename(mailbox, META_CACHE);
+ repack->i.options &= ~(OPT_MAILBOX_NEEDS_REPACK|OPT_MAILBOX_NEEDS_UNLINK);
- return r;
+ return mailbox_repack_commit(&repack);
fail:
- if (newcache_fd != -1) close(newcache_fd);
- if (newindex_fd != -1) close(newindex_fd);
- return IMAP_IOERROR;
+ mailbox_repack_abort(&repack);
+ return r;
}
/*
diff --git a/imap/mailbox.h b/imap/mailbox.h
index 80001e1..8e9e9fb 100644
--- a/imap/mailbox.h
+++ b/imap/mailbox.h
@@ -498,17 +498,25 @@ extern int mailbox_reconstruct(const char *name, int flags);
extern int mailbox_index_recalc(struct mailbox *mailbox);
/* for upgrade index */
+extern int mailbox_open_index(struct mailbox *mailbox);
extern int mailbox_buf_to_index_record(const char *buf,
struct index_record *record);
-extern int mailbox_open_index(struct mailbox *mailbox);
-extern int mailbox_buf_to_index_header(struct index_header *i, const char *buf);
-extern bit32 mailbox_index_header_to_buf(struct index_header *i,
- unsigned char *buf);
-extern void mailbox_index_update_counts(struct mailbox *mailbox,
- struct index_record *record,
- int is_add);
-extern bit32 mailbox_index_record_to_buf(struct index_record *record,
- unsigned char *buf);
+extern int mailbox_buf_to_index_header(const char *buf,
+ struct index_header *i);
+
+/* for repack */
+struct mailbox_repack {
+ struct mailbox *mailbox;
+ struct index_header i;
+ int newindex_fd;
+ int newcache_fd;
+};
+extern int mailbox_repack_setup(struct mailbox *mailbox,
+ struct mailbox_repack **repackptr);
+extern int mailbox_repack_add(struct mailbox_repack *repack,
+ struct index_record *record);
+extern void mailbox_repack_abort(struct mailbox_repack **repackptr);
+extern int mailbox_repack_commit(struct mailbox_repack **repackptr);
#endif /* INCLUDED_MAILBOX_H */
diff --git a/imap/upgrade_index.c b/imap/upgrade_index.c
index 59e3044..d871b73 100644
--- a/imap/upgrade_index.c
+++ b/imap/upgrade_index.c
@@ -85,112 +85,46 @@
#include "sequence.h"
#include "xmalloc.h"
-static int sort_record(const void *a, const void *b)
-{
- struct index_record *ra = (struct index_record *)a;
- struct index_record *rb = (struct index_record *)b;
- return ra->uid - rb->uid;
-}
+struct expunge_data {
+ uint32_t uid;
+ const char *base;
+};
-static int update_record_from_cache(struct mailbox *mailbox,
- struct index_record *record)
+static int sort_expunge(const void *a, const void *b)
{
- int r;
- bit32 crc;
-
- r = mailbox_open_cache(mailbox);
- if (r) return r;
-
- if (!record->cache_offset)
- return IMAP_IOERROR;
-
- r = cache_parserecord(&mailbox->cache_buf,
- record->cache_offset, &record->crec);
- if (r) return r;
-
- crc = crc32_buf(cache_buf(record));
- if (record->cache_crc) {
- if (crc != record->cache_crc)
- return IMAP_MAILBOX_CRC;
- }
- else {
- record->cache_crc = crc;
- }
-
- /* extract the date for GMTIME field */
- if (cacheitem_size(record, CACHE_ENVELOPE) > 2) {
- char *envtokens[NUMENVTOKENS];
- char *tmpenv = xstrndup(cacheitem_base(record, CACHE_ENVELOPE) + 1,
- cacheitem_size(record, CACHE_ENVELOPE) - 2);
- parse_cached_envelope(tmpenv, envtokens, VECTOR_SIZE(envtokens));
- record->gmtime = message_parse_date(envtokens[ENV_DATE],
- PARSE_TIME|PARSE_ZONE|PARSE_NOCREATE|PARSE_GMT);
- free(tmpenv);
- }
- else {
- /* better than nothing! */
- record->gmtime = record->internaldate;
- }
-
- return 0;
+ struct expunge_data *ea = (struct expunge_data *)a;
+ struct expunge_data *eb = (struct expunge_data *)b;
+ return ea->uid - eb->uid;
}
static void upgrade_index_record(struct mailbox *mailbox,
const char *buf,
- int old_version,
struct index_record *record,
int record_size)
{
- indexbuffer_t rbuf;
- char *recordbuf = (char *)rbuf.buf;
- int recalc = 0;
+ char recordbuf[INDEX_RECORD_SIZE];
struct utimbuf settime;
const char *fname;
+ assert(record_size <= INDEX_RECORD_SIZE);
+
memset(recordbuf, 0, INDEX_RECORD_SIZE);
- if (INDEX_RECORD_SIZE < record_size)
- memcpy(recordbuf, buf, INDEX_RECORD_SIZE);
- else
- memcpy(recordbuf, buf, record_size);
-
- /* CONTENT_LINES added with minor version 5 */
- /* CACHE_VERSION added with minor version 6 */
-
- /* 12-byte GUIDs added with minor version 7 */
- /* GUIDs extended from 12 to 20 bytes with minor version 10 */
- if (old_version < 10)
- recalc = 1;
- else {
- /* if it's all zeros for the final 8 bits, it was probably upgraded
- * and also needs recalculation */
- if (ntohl(*((bit32 *)(recordbuf+OFFSET_MESSAGE_GUID+12))) == 0 &&
- ntohl(*((bit32 *)(recordbuf+OFFSET_MESSAGE_GUID+16))) == 0)
- recalc = 1;
- }
+ memcpy(recordbuf, buf, record_size);
/* do the initial parse. Ignore the result, crc32 will mismatch
* for sure */
mailbox_buf_to_index_record(recordbuf, record);
- if (!recalc && old_version < 12) {
- /* let's try a cheaper upgrade option - just reading the
- cache record for details */
- if (update_record_from_cache(mailbox, record))
- recalc = 1;
- }
-
fname = mailbox_message_fname(mailbox, record->uid);
- if (recalc) {
- if (message_parse(fname, record)) {
- /* failed to create, don't try to write */
- record->crec.len = 0;
- /* and the record is expunged too! */
- record->system_flags |= FLAG_EXPUNGED | FLAG_UNLINKED;
- syslog(LOG_ERR, "IOERROR: FATAL - failed to parse "
- "file %s for upgrade, expunging", fname);
- return;
- }
+ if (message_parse(fname, record)) {
+ /* failed to create, don't try to write */
+ record->crec.len = 0;
+ /* and the record is expunged too! */
+ record->system_flags |= FLAG_EXPUNGED | FLAG_UNLINKED;
+ syslog(LOG_ERR, "IOERROR: FATAL - failed to parse "
+ "file %s for upgrade, expunging", fname);
+ return;
}
/* update the mtime to match the internaldate */
@@ -204,25 +138,27 @@ static void upgrade_index_record(struct mailbox *mailbox,
int upgrade_index(struct mailbox *mailbox)
{
uint32_t recno, erecno;
+ uint32_t uid, euid;
unsigned long oldmapnum;
unsigned long oldnum_records;
unsigned long expunge_num = 0;
- unsigned uid = 0;
- bit32 oldminor_version, oldstart_offset, oldrecord_size;
+ uint32_t oldminor_version, oldstart_offset, oldrecord_size;
indexbuffer_t headerbuf;
- indexbuffer_t recordbuf;
const char *bufp = NULL;
char *hbuf = (char *)headerbuf.buf;
- char *rbuf = (char *)recordbuf.buf;
- int newindex_fd = -1;
const char *fname;
const char *datadirname;
struct stat sbuf;
struct seqset *seq = NULL;
struct index_record record;
- struct index_record *expunge_records = NULL;
- struct index_record *recordptr;
- int r, n;
+ int expunge_fd = -1;
+ const char *expunge_base = NULL;
+ unsigned long expunge_len = 0; /* mapped size */
+ unsigned long emapnum;
+ bit32 eversion, eoffset, expungerecord_size;
+ struct expunge_data *expunge_data = NULL;
+ struct mailbox_repack *repack = NULL;
+ int r;
if (mailbox->index_size < OFFSET_NUM_RECORDS)
return IMAP_MAILBOX_BADFORMAT;
@@ -276,7 +212,7 @@ int upgrade_index(struct mailbox *mailbox)
}
/* ignore the result - we EXPECT a CRC32 mismatch */
- mailbox_buf_to_index_header(&mailbox->i, hbuf);
+ mailbox_buf_to_index_header(hbuf, &mailbox->i);
/* HIGHESTMODSEQ[_64] added with minor version 8 */
if (oldminor_version < 8)
@@ -301,13 +237,7 @@ int upgrade_index(struct mailbox *mailbox)
if (oldminor_version < 12) {
struct seen *seendb;
struct seendata sd;
- unsigned long erecno;
- unsigned long emapnum;
- bit32 eversion, eoffset, esize;
char *owner_userid;
- int expunge_fd = -1;
- const char *expunge_base = NULL;
- unsigned long expunge_len = 0; /* mapped size */
/* remove the CONDSTORE option - it's implicit now */
mailbox->i.options &= ~OPT_IMAP_CONDSTORE;
@@ -326,8 +256,6 @@ int upgrade_index(struct mailbox *mailbox)
mailbox->i.first_expunged = 0;
/* we can't know about deletions before the current modseq */
mailbox->i.deletedmodseq = mailbox->i.highestmodseq;
- /* we're repacking now! */
- mailbox->i.last_repack_time = time(NULL);
/* bootstrap CRC matching */
mailbox->i.header_file_crc = mailbox->header_file_crc;
@@ -369,61 +297,38 @@ int upgrade_index(struct mailbox *mailbox)
* versions are skewed for some reason */
eversion = ntohl(*((bit32 *)(expunge_base+OFFSET_MINOR_VERSION)));
eoffset = ntohl(*((bit32 *)(expunge_base+OFFSET_START_OFFSET)));
- esize = ntohl(*((bit32 *)(expunge_base+OFFSET_RECORD_SIZE)));
- expunge_num = ntohl(*((bit32 *)(expunge_base+OFFSET_NUM_RECORDS)));
+ expungerecord_size = ntohl(*((bit32 *)(expunge_base+OFFSET_RECORD_SIZE)));
/* bogus data at the start of the expunge file? */
- if (!eoffset || !esize)
+ if (!eoffset || !expungerecord_size)
goto no_expunge;
- expunge_records = xmalloc(expunge_num * sizeof(struct index_record));
- emapnum = (sbuf.st_size - eoffset) / esize;
+ expunge_num = ntohl(*((bit32 *)(expunge_base+OFFSET_NUM_RECORDS)));
+ emapnum = (sbuf.st_size - eoffset) / expungerecord_size;
if (emapnum < expunge_num) {
syslog(LOG_ERR, "IOERROR: %s map doesn't fit, shrinking expunge %lu to %lu",
mailbox->name, expunge_num, emapnum);
expunge_num = emapnum;
}
+ /* make sure there's space for them */
+ expunge_data = xmalloc(expunge_num * sizeof(struct expunge_data));
+
+ /* find the start offset for each record, and the UID */
for (erecno = 1; erecno <= expunge_num; erecno++) {
- struct index_record *record = &expunge_records[erecno-1];
- bufp = expunge_base + eoffset + (erecno-1)*esize;
- upgrade_index_record(mailbox, bufp, eversion, record, esize);
- record->system_flags |= FLAG_EXPUNGED;
- if (!mailbox->i.first_expunged ||
- mailbox->i.first_expunged > record->last_updated)
- mailbox->i.first_expunged = record->last_updated;
+ bufp = expunge_base + eoffset + (erecno-1)*expungerecord_size;
+ expunge_data[erecno-1].uid = ntohl(*((bit32 *)(bufp+OFFSET_UID)));
+ expunge_data[erecno-1].base = bufp;
}
/* expunge files were not sorted. So sort them now for easier
* interleaving */
- qsort(expunge_records, expunge_num,
- sizeof(struct index_record), &sort_record);
-
-no_expunge:
- if (expunge_fd != -1) close(expunge_fd);
- if (expunge_base) map_free(&expunge_base, &expunge_len);
+ qsort(expunge_data, expunge_num,
+ sizeof(struct expunge_data), &sort_expunge);
}
+no_expunge:
- /* update buffer with upgraded values */
- mailbox_index_header_to_buf(&mailbox->i, (unsigned char *)hbuf);
-
- /* open the new index file */
- fname = mailbox_meta_newfname(mailbox, META_INDEX);
- newindex_fd = open(fname, O_RDWR|O_TRUNC|O_CREAT, 0666);
- if (newindex_fd == -1) goto fail;
-
- /* Write new header - first pass only */
- n = retry_write(newindex_fd, hbuf, INDEX_HEADER_SIZE);
- if (n == -1) goto fail;
-
- /* initialise counters */
- mailbox->i.quota_mailbox_used = 0;
- mailbox->i.num_records = 0;
- mailbox->i.sync_crc = 0; /* no records is blank */
- mailbox->i.answered = 0;
- mailbox->i.deleted = 0;
- mailbox->i.flagged = 0;
- mailbox->i.exists = 0;
+ mailbox_repack_setup(mailbox, &repack);
/* Write the rest of new index */
recno = 1;
@@ -434,57 +339,39 @@ no_expunge:
bufp = mailbox->index_base + oldstart_offset + (recno-1)*oldrecord_size;
uid = ntohl(*((bit32 *)(bufp+OFFSET_UID)));
}
-
- /* case: only expunge records left */
- if (recno > oldnum_records) {
- recordptr = &expunge_records[erecno-1];
- erecno++;
+ else {
+ uid = UINT32_MAX;
+ }
+ if (erecno <= expunge_num) {
+ euid = expunge_data[erecno-1].uid;
+ }
+ else {
+ euid = UINT32_MAX;
}
- /* case: index record is lower uid */
- else if (erecno > expunge_num || uid <= expunge_records[erecno-1].uid) {
- upgrade_index_record(mailbox, bufp, oldminor_version, &record,
- oldrecord_size);
+ /* case: index UID is first, or the same */
+ if (uid <= euid) {
+ upgrade_index_record(mailbox, bufp, &record, oldrecord_size);
recno++;
- if (erecno <= expunge_num && uid == expunge_records[erecno-1].uid)
- erecno++; /* duplicate UID - skip expunge record */
- recordptr = &record;
+ if (uid == euid) /* duplicate in both, skip expunged */
+ erecno++;
}
-
- /* case: expunge record is lower uid */
else {
- recordptr = &expunge_records[erecno-1];
+ upgrade_index_record(mailbox, expunge_data[erecno-1].base,
+ &record, expungerecord_size);
+ record.system_flags |= FLAG_EXPUNGED;
erecno++;
}
- if (oldminor_version < 12 && seqset_ismember(seq, recordptr->uid))
- recordptr->system_flags |= FLAG_SEEN;
+ if (oldminor_version < 12 && seqset_ismember(seq, record.uid))
+ record.system_flags |= FLAG_SEEN;
- /* write the cache record if necessary */
- r = mailbox_append_cache(mailbox, recordptr);
+ r = mailbox_repack_add(repack, &record);
if (r) goto fail;
-
- mailbox_index_update_counts(mailbox, recordptr, 1);
- mailbox_index_record_to_buf(recordptr, (unsigned char *)rbuf);
-
- n = retry_write(newindex_fd, rbuf, INDEX_RECORD_SIZE);
- if (n == -1) goto fail;
-
- mailbox->i.num_records++;
}
- mailbox_index_header_to_buf(&mailbox->i, (unsigned char *)hbuf);
-
- lseek(newindex_fd, 0L, SEEK_SET);
- n = retry_write(newindex_fd, hbuf, INDEX_HEADER_SIZE);
- if (n == -1) goto fail;
-
- r = fsync(newindex_fd);
- if (r == -1) goto fail;
- close(newindex_fd);
-
- r = mailbox_meta_rename(mailbox, META_INDEX);
- if (r == -1) goto fail;
+ r = mailbox_repack_commit(&repack);
+ if (r) goto fail;
/* don't need this file any more! */
unlink(mailbox_meta_fname(mailbox, META_EXPUNGE));
@@ -495,16 +382,10 @@ no_expunge:
oldminor_version, MAILBOX_MINOR_VERSION);
done:
+ if (expunge_fd != -1) close(expunge_fd);
+ if (expunge_base) map_free(&expunge_base, &expunge_len);
seqset_free(seq);
- free(expunge_records);
-
- /* commit the cache first so it doesn't stay "dirty" */
- mailbox_commit_cache(mailbox);
-
- /* special case, completely forgiven from being clean... */
- mailbox->i.dirty = 0;
- mailbox->quota_dirty = 0;
- mailbox->modseq_dirty = 0;
+ free(expunge_data);
/* it's definitely changed! */
r = mailbox_open_index(mailbox);
@@ -513,9 +394,12 @@ done:
return 0;
fail:
- if (newindex_fd != -1) close(newindex_fd);
+ if (expunge_fd != -1) close(expunge_fd);
+ if (expunge_base) map_free(&expunge_base, &expunge_len);
seqset_free(seq);
- free(expunge_records);
+ free(expunge_data);
+
+ mailbox_repack_abort(&repack);
syslog(LOG_ERR, "Index upgrade failed: %s", mailbox->name);
--
1.7.1
----
Cyrus Home Page: http://www.cyrusimap.org/
List Archives/Info: http://lists.andrew.cmu.edu/pipermail/info-cyrus/
