Paolo Cravero wrote:
PC> I have read in the docs that the autocreate patch does not work
PC> with murder, and I have experienced it too. But...
PC> ... if I leave the autocreate ON on backends only, and then
PC> simulate an IMAP access or mail delivery right at the backend
PC> level, I should not get into troubles, right?
You might be interested in the attached patch, which is something I
cooked up last year for our internal Cyrus builds. It's a fairly dirty
hack to provide frontends with a way to decide which backend should they
go to for a mailbox that doesn't yet exist.
The basic idea is to catch lookups by proxyd and lmtpproxyd which would
normally return IMAP_MAILBOX_NONEXISTENT and call out to an
administrator-defined helper script (roughly modelled on the way Squid
auth helpers work) which decides whether and where to assume the mailbox
should exist. Then normal auto* on the backends kicks in...
There are many ways in which this could be improved. Most obviously
admins shouldn't be required to maintain the helper script across all
the FEs - the decision logic should be at a central location such as the
MUPDATE host.
And it would be good to extend the information passed to the helper (eg
authenticated sender info in the LMTP case), and back (eg a way to set
mailbox ACLs or other config). The latter would require much better
integration with the UoA patches. The protocol spoken between the Cyrus
components and the helper app should resemble some sane IMAP-related one...
Ideas and improvements welcome ;-)
Cheers
Duncan
--
Duncan Gibb - Technical Director
Sirius Corporation plc - control through freedom
http://www.siriusit.co.uk/
t: +44 870 608 0063 || m: +44 7977 441 515
#! /bin/sh /usr/share/dpatch/dpatch-run
## 95-automurder-frontend.dpatch by Duncan Gibb <duncan.gibb@xxxxxxxxxxxxxx>
##
## All lines beginning with `## DP:' are a description of the patch.
## DP: Run a helper application on frontends which can decide whether to
## DP: pretend mailboxes exist on a backend - and if so on which - in order to
## DP: trigger autocreate
@DPATCH@
diff -Nrub --exclude '*svn*' --exclude '*~' --exclude debian reference/imap/automurder.c editing/imap/automurder.c
--- reference/imap/automurder.c 1970-01-01 01:00:00.000000000 +0100
+++ editing/imap/automurder.c 2009-05-10 10:01:09.000000000 +0100
@@ -0,0 +1,259 @@
+/* automurder.c
+ *
+ * Use an external helper to decide which backend should
+ * to assume a non-existant mailbox onto.
+ *
+ * Copyright 2008 Duncan Gibb, Sirius Corporation plc
+ */
+
+#include <config.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <syslog.h>
+#include <sys/time.h>
+#include <signal.h>
+#include <errno.h>
+
+#include "automurder.h"
+#include "imap_err.h"
+#include "libconfig.h"
+#include "mboxlist.h"
+#include "xmalloc.h"
+
+#define BUFSIZE MAX_MAILBOX_NAME+2
+
+void spawn_helper(void);
+void shutdown_helper(void);
+void fault(void);
+
+static int helper_pid = 0;
+static int qfd, afd;
+static int respawn_count;
+static int fault_count;
+
+static char automurder_last_refused_mbox[BUFSIZE];
+
+void automurder_init(void)
+{
+ respawn_count = 0;
+ automurder_last_refused_mbox[0] = '\0';
+
+ if(config_getswitch(IMAPOPT_AUTOMURDER_HELPER_PREFORK)) {
+ spawn_helper();
+ }
+}
+
+void automurder_shutdown(void)
+{
+ shutdown_helper();
+}
+
+void fault(void)
+{
+ if( ++fault_count > config_getint(IMAPOPT_AUTOMURDER_HELPER_FAULT_TOLLERANCE) ) {
+ syslog(LOG_ERR, "automurder helper fault tollerance reached. Respawning helper.");
+ shutdown_helper();
+ spawn_helper();
+ }
+}
+
+void spawn_helper(void)
+{
+ int helper_stdin;
+ int helper_stdout;
+ int mypipe[2];
+
+ if(!config_getswitch(IMAPOPT_AUTOMURDER_FRONTEND))
+ return;
+
+ if(!config_getstring(IMAPOPT_AUTOMURDER_HELPER)) {
+ syslog(LOG_ERR, "automurder_frontend enabled without configuring helper application");
+ return;
+ }
+
+ if(respawn_count++ > config_getint(IMAPOPT_AUTOMURDER_HELPER_MAXRESPAWN)) {
+ return;
+ } else if(respawn_count == config_getint(IMAPOPT_AUTOMURDER_HELPER_MAXRESPAWN)) {
+ syslog(LOG_ERR, "automurder_helper_maxrespawn reached.");
+ return;
+ }
+
+ if(pipe (mypipe)) {
+ syslog(LOG_ERR, "Could not create question pipe for automurder helper.");
+ return;
+ }
+
+ helper_stdin = mypipe[0];
+ qfd = mypipe[1];
+
+ if(pipe (mypipe)) {
+ syslog(LOG_ERR, "Could not create answer pipe for automurder helper.");
+ return;
+ }
+
+ afd = mypipe[0];
+ helper_stdout = mypipe[1];
+
+ fcntl( afd, F_SETFL, O_NONBLOCK );
+
+ fault_count = 0;
+
+ helper_pid=fork();
+
+ if(helper_pid == -1) {
+ syslog(LOG_ERR, "Could not fork to create automurder helper.");
+ return;
+ }
+
+ if(helper_pid == 0) {
+ const char *av = NULL;
+ const char *env = NULL;
+
+ dup2( helper_stdin, 0 );
+ dup2( helper_stdout, 1 );
+
+ execve( config_getstring(IMAPOPT_AUTOMURDER_HELPER), &av, &env );
+
+ /* Should not return */
+
+ syslog(LOG_ERR, "Failed to exec() automurder helper '%s'.",
+ config_getstring(IMAPOPT_AUTOMURDER_HELPER));
+
+ _exit(errno);
+ }
+}
+
+void shutdown_helper(void)
+{
+ if(helper_pid)
+ kill(helper_pid, SIGKILL);
+
+ if(afd)
+ close(afd);
+
+ if(qfd)
+ close(qfd);
+
+ helper_pid = 0;
+}
+
+/* Callout to an administrator-defined helper program which establishes
+ * which backend a given mailbox ought to be created on. Only useful
+ * if the backend has autocreateonpost enabled.
+ * The command defined by automurder_helper should return a
+ * string of the form "OK servername" if the mailbox should be created
+ * or "BAD human-understandable reason"
+ * A non-OK response or silence for automurder_helper_timeout seconds
+ * will return IMAP_MAILBOX_NONEXISTENT. Otherwise 'server' will be
+ * populated.
+ */
+
+int automurder_lookup(const char *name, char **server, char **aclp)
+{
+ static char servername[HOSTNAME_SIZE+1];
+ char buf[BUFSIZE];
+ fd_set fds;
+ struct timeval tv;
+ int r;
+
+ if(aclp) *aclp = "";
+
+ if(!server) {
+ syslog(LOG_DEBUG, "automurder_lookup called for mailbox %s without a server", name);
+ return IMAP_INTERNAL;
+ }
+
+ /* Some clients won't take "no" for an answer */
+ if(!strcmp(name, automurder_last_refused_mbox)) {
+ return IMAP_MAILBOX_NONEXISTENT;
+ }
+
+ if(!helper_pid) {
+ spawn_helper();
+ }
+
+ /* Throw away any pending input */
+ errno = 0;
+ while( read( afd, buf, BUFSIZE ) && (!errno) );
+
+ if( errno != EAGAIN ) {
+ syslog(LOG_ERR, "Error emptying the automurder helper answer pipe." );
+ fault();
+ return IMAP_SERVER_UNAVAILABLE;
+ }
+
+ if( snprintf( buf, BUFSIZE , "%s\n", name ) > BUFSIZE ) {
+ syslog(LOG_ERR, "Error composing automurder question - mailbox name '%s' too long.", name);
+ return IMAP_MAILBOX_BADNAME;
+ }
+
+ if( write( qfd, buf, strlen(buf) ) == -1) {
+ syslog(LOG_ERR, "Error writing to automurder helper for mailbox '%s'.", name);
+ fault();
+ return IMAP_SERVER_UNAVAILABLE;
+ }
+
+ FD_ZERO( &fds );
+ FD_SET( afd, &fds );
+ tv.tv_sec = config_getint(IMAPOPT_AUTOMURDER_HELPER_TIMEOUT);
+ tv.tv_usec = 0;
+
+ select( afd+1, &fds, NULL, NULL, &tv );
+ if( !FD_ISSET( afd, &fds )) {
+ syslog(LOG_ERR, "Automurder lookup timed out for mailbox '%s'.", name);
+ fault();
+ return IMAP_SERVER_UNAVAILABLE;
+ }
+
+ buf[0] = '\0';
+ if( (r = read( afd, buf, BUFSIZE )) == -1 ) {
+ syslog(LOG_ERR, "Error reading from automurder helper for mailbox '%s'.", name);
+ fault();
+ return IMAP_SERVER_UNAVAILABLE;
+ }
+ buf[r] = '\0'; /* We're going to do string operations on a partial buffer */
+
+ if(!(strncmp( buf, "OK ", 3 ))) {
+ int i;
+ char c;
+ for( i = 0; i < HOSTNAME_SIZE && i < (r-3); i++ ) {
+ c = servername[i] = buf[i+3];
+ if( c == ' ' || c == '\n' || c == '\r' || c == '\t' ) {
+ servername[i] = '\0';
+ break;
+ }
+ }
+
+ syslog(LOG_INFO, "Automurder helper approved creation of '%s' on backend '%s'.",
+ name, servername);
+
+ *server = servername;
+ return 0;
+ }
+
+ if(!(strncmp( buf, "BAD " , 4 ))) {
+ int i;
+ for( i = 4; i < BUFSIZE; i++ ) {
+ if( buf[i] == '\n' || buf[i] == '\r' || buf[i] == '\0' ) {
+ buf[i] = '\0';
+ break;
+ }
+ }
+ syslog(LOG_INFO, "Automurder helper denied creation of '%s' because '%s'.",
+ name, buf+4);
+ strcpy(automurder_last_refused_mbox, name);
+ return IMAP_MAILBOX_NONEXISTENT;
+ }
+
+ syslog(LOG_ERR, "Automurder helper protocol violation: '%s' returned '%s'.",
+ name, buf);
+
+ /* We don't tollerate protocol violations. Just kill the helper. */
+ shutdown_helper();
+ spawn_helper();
+
+ return IMAP_SERVER_UNAVAILABLE;
+}
+
diff -Nrub --exclude '*svn*' --exclude '*~' --exclude debian reference/imap/automurder.h editing/imap/automurder.h
--- reference/imap/automurder.h 1970-01-01 01:00:00.000000000 +0100
+++ editing/imap/automurder.h 2009-05-09 13:04:58.000000000 +0100
@@ -0,0 +1,16 @@
+/* automurder.h
+ *
+ * Use an external helper to decide which backend should
+ * to assume a non-existant mailbox onto.
+ *
+ * Copyright 2008 Duncan Gibb, Sirius Corporation plc
+ */
+
+#ifndef INCLUDED_AUTOMURDER_H
+#define INCLUDED_AUTOMURDER_H
+
+void automurder_init(void);
+void automurder_shutdown(void);
+int automurder_lookup(const char *name, char **server, char **aclp);
+
+#endif /* INCLUDED_AUTOMURDER_H */
diff -Nrub --exclude '*svn*' --exclude '*~' --exclude debian reference/imap/imapd.c editing/imap/imapd.c
--- reference/imap/imapd.c 2009-05-09 13:04:40.000000000 +0100
+++ editing/imap/imapd.c 2009-05-09 13:04:58.000000000 +0100
@@ -73,6 +73,7 @@
#include "annotate.h"
#include "append.h"
#include "auth.h"
+#include "automurder.h"
#include "backend.h"
#include "charset.h"
#include "exitcodes.h"
@@ -476,6 +477,11 @@
r = mboxlist_detail(name, &mbtype, pathp, mpathp, &remote, &acl, tid);
}
+ if (r == IMAP_MAILBOX_NONEXISTENT && config_getswitch(IMAPOPT_AUTOMURDER_FRONTEND)) {
+ r = automurder_lookup( name, &remote, &acl );
+ mbtype = MBTYPE_REMOTE;
+ }
+
if(partp) *partp = remote;
if(aclp) *aclp = acl;
if(flags) *flags = mbtype;
@@ -697,6 +703,8 @@
statuscache_open(NULL);
}
+ automurder_init();
+
/* Create a protgroup for input from the client and selected backend */
protin = protgroup_new(2);
@@ -819,6 +827,7 @@
/* Called by service API to shut down the service */
void service_abort(int error)
{
+ automurder_shutdown();
shut_down(error);
}
diff -Nrub --exclude '*svn*' --exclude '*~' --exclude debian reference/imap/lmtpd.c editing/imap/lmtpd.c
--- reference/imap/lmtpd.c 2009-05-09 13:04:40.000000000 +0100
+++ editing/imap/lmtpd.c 2009-05-09 13:04:58.000000000 +0100
@@ -70,6 +70,7 @@
#include "append.h"
#include "assert.h"
#include "auth.h"
+#include "automurder.h"
#include "backend.h"
#include "duplicate.h"
#include "exitcodes.h"
@@ -195,6 +196,7 @@
config_mupdate_server, error_message(r));
fatal("error connecting with MUPDATE server", EC_TEMPFAIL);
}
+ automurder_init();
}
else {
dupelim = config_getswitch(IMAPOPT_DUPLICATESUPPRESSION);
@@ -329,6 +331,7 @@
/* Called by service API to shut down the service */
void service_abort(int error)
{
+ automurder_shutdown();
shut_down(error);
}
@@ -433,6 +436,9 @@
r = mupdate_find(mhandle, name, &mailboxdata);
if (r == MUPDATE_MAILBOX_UNKNOWN) {
+ if(config_getswitch(IMAPOPT_AUTOMURDER_FRONTEND)) {
+ return automurder_lookup( name, server, aclp );
+ }
return IMAP_MAILBOX_NONEXISTENT;
} else if (r) {
/* xxx -- yuck: our error handling for now will be to exit;
diff -Nrub --exclude '*svn*' --exclude '*~' --exclude debian reference/imap/Makefile.in editing/imap/Makefile.in
--- reference/imap/Makefile.in 2009-05-09 13:04:40.000000000 +0100
+++ editing/imap/Makefile.in 2009-05-09 13:04:58.000000000 +0100
@@ -102,7 +102,7 @@
annotate.o search_engines.o squat.o squat_internal.o mbdump.o \
imapparse.o telemetry.o user.o notify.o idle.o quota_db.o \
sync_log.o autosieve.o $(SEEN) mboxkey.o backend.o tls.o tls_th-lock.o \
- message_guid.o statuscache_db.o
+ message_guid.o statuscache_db.o automurder.o
IMAPDOBJS=pushstats.o imapd.o proxy.o imap_proxy.o index.o version.o
@@ -187,21 +187,21 @@
$(CC) $(LDFLAGS) -o idled \
idled.o mutex_fake.o libimap.a $(SIEVE_LIBS) $(DEPLIBS) $(LIBS)
-lmtpd: lmtpd.o proxy.o $(LMTPOBJS) $(SIEVE_OBJS) mutex_fake.o \
+lmtpd: lmtpd.o proxy.o automurder.o $(LMTPOBJS) $(SIEVE_OBJS) mutex_fake.o \
libimap.a $(SIEVE_LIBS) $(DEPLIBS) $(SERVICE)
$(CC) $(LDFLAGS) -o lmtpd \
- $(SERVICE) lmtpd.o proxy.o $(LMTPOBJS) $(SIEVE_OBJS) \
+ $(SERVICE) lmtpd.o proxy.o automurder.o $(LMTPOBJS) $(SIEVE_OBJS) \
mutex_fake.o libimap.a $(SIEVE_LIBS) $(DEPLIBS) $(LIBS) $(LIB_WRAP)
-lmtpd.pure: lmtpd.o proxy.o $(LMTPOBJS) $(SIEVE_OBJS) \
+lmtpd.pure: lmtpd.o proxy.o automurder.o $(LMTPOBJS) $(SIEVE_OBJS) \
mutex_fake.o libimap.a $(SIEVE_LIBS) $(DEPLIBS) $(SERVICE)
$(PURIFY) $(PUREOPT) $(CC) $(LDFLAGS) -o lmtpd.pure \
- $(SERVICE) lmtpd.o proxy.o $(LMTPOBJS) $(SIEVE_OBJS) \
+ $(SERVICE) lmtpd.o proxy.o automurder.o $(LMTPOBJS) $(SIEVE_OBJS) \
mutex_fake.o libimap.a $(SIEVE_LIBS) $(DEPLIBS) $(LIBS) $(LIB_WRAP)
-imapd: $(IMAPDOBJS) mutex_fake.o libimap.a $(SIEVE_LIBS) $(DEPLIBS) $(SERVICE)
+imapd: $(IMAPDOBJS) mutex_fake.o libimap.a automurder.o $(SIEVE_LIBS) $(DEPLIBS) $(SERVICE)
$(CC) $(LDFLAGS) -o imapd \
- $(SERVICE) $(IMAPDOBJS) mutex_fake.o \
+ $(SERVICE) $(IMAPDOBJS) mutex_fake.o automurder.o \
libimap.a $(SIEVE_LIBS) $(DEPLIBS) $(LIBS) $(LIB_WRAP)
imapd.pure: $(IMAPDOBJS) mutex_fake.o libimap.a $(SIEVE_LIBS) $(DEPLIBS) $(SERVICE)
diff -Nrub --exclude '*svn*' --exclude '*~' --exclude debian reference/lib/imapoptions editing/lib/imapoptions
--- reference/lib/imapoptions 2009-05-09 13:04:40.000000000 +0100
+++ editing/lib/imapoptions 2009-05-09 13:04:58.000000000 +0100
@@ -203,6 +203,39 @@
creating the mailbox INBOX. The user's quota is set to the value
if it is positive, otherwise the user has unlimited quota. */
+{ "automurder_frontend", 0, SWITCH }
+/* If non-zero, ask automurder_helper which backend to assume
+ non-existant mailboxes are located on. Only useful where backends
+ have autocreate enabled. */
+
+{ "automurder_helper", NULL, STRING }
+/* Helper application to use if automurder_frontend is enabled.
+ The command will receive newline-separated mailbox names on
+ stdin and should print "OK <servername>\n" to stdout if the
+ mailbox should be assumed to exist, where <servername> is the
+ name of the appropriate backend server. Otherwise it should
+ print "BAD human-understandable reason".
+ The helper will be killed and restarted if it prints anything else.
+ Only useful if the backend has autocreate enabled.
+ Note that this command is executed with the priviledges of the cyrus
+ user. */
+
+{ "automurder_helper_fault_tollerance", 5, INT }
+/* Number of lookup helper faults to tollerate before respawning it. */
+
+{ "automurder_helper_maxrespawn", 5, INT }
+/* Number of times to tollerate restarting a lookup helper before
+ abandoning it. */
+
+{ "automurder_helper_prefork", 0, SWITCH }
+/* If true, imapd and lmtpd will prefork automurder_helper when
+ they start. This causes a lot of process table clutter, so
+ only use it if the helper needs to do some complex setup. */
+
+{ "automurder_helper_timeout", 5, INT }
+/* Number of seconds to allow for automurder_lookup_command to respond
+ before assuming the possible mailbox should not exist. */
+
{ "berkeley_cachesize", 512, INT }
/* Size (in kilobytes) of the shared memory buffer pool (cache) used
by the berkeley environment. The minimum allowed value is 20. The
----
Cyrus Home Page: http://cyrusimap.web.cmu.edu/
Cyrus Wiki/FAQ: http://cyrusimap.web.cmu.edu/twiki
List Archives/Info: http://asg.web.cmu.edu/cyrus/mailing-list.html
