On Fri, Nov 21, 2008 at 02:27:07PM -0500, Anthony Tibbs wrote:
> Hi Eddy,
>
> I had worked on this briefly and, given a day or so, I could probably come
> up with a working solution at least for byte/traffic accounting. Logging
> 'messages retrieved' for POP3 could make some sense, but it gets complicated
> with clients using "TOP" to partially retrieve messages for inspection, and
> in the IMAP world its all but meaningless (since that might involve copies
> to folders, etc.).
>
> For pure traffic accounting purposes, though, that isn't so hard,
> particularly if you don't want it saved in a database anywhere, but merely
> added as a log entry.
We have a patch at fastmail that does rate logging - but we have some
tricky requirements that it doesn't log meta traffic, just body
transfers, and only some of those. It's a bit weird.
I suspect it would actually be _easier_ to just do byte counting in
prot_write depending on the filehandle.
Actually, why not just do traffic counting on all protstreams? It's not
that hard.
The attached patch adds traffic counting to pop3 and imap connections,
it prints total bytes in and out at shutdown or reset. I've also
refactored our "auditlog" patch on top of this so that the traffic data
gets logged along with the sessionid allowing it to be easily tied back
to a login (you can do it with PID from the log file if you want, but
we use the session_id so we can track it back to the proxy logs on the
frontends and there to the source IP and specific login used - because
we allow our users to set up multiple passwords for their account with
different restrictions on use)
Counting messages would be a separate patch to just pop3.
Bron ( hey, I was interested. It took about an hour and compiled first
try! )
Index: cyrus-imapd-2.3.13/lib/prot.c
===================================================================
--- cyrus-imapd-2.3.13.orig/lib/prot.c 2008-11-22 21:48:41.000000000 +1100
+++ cyrus-imapd-2.3.13/lib/prot.c 2008-11-22 22:10:22.000000000 +1100
@@ -103,6 +103,9 @@
if(write)
newstream->cnt = PROT_BUFSIZE;
+ newstream->bytes_in = 0;
+ newstream->bytes_out = 0;
+
return newstream;
}
@@ -344,6 +347,7 @@
s->cnt = 0;
s->error = 0;
s->eof = 0;
+ s->bytes_in = 0;
return 0;
}
@@ -545,6 +549,7 @@
}
s->cnt--; /* we return the first char */
+ s->bytes_in++;
return *s->buf;
}
} while (1);
@@ -866,6 +871,7 @@
memcpy(s->ptr, buf, len);
s->ptr += len;
s->cnt -= len;
+ s->bytes_out += len;
if (s->error || s->eof) return EOF;
assert(s->cnt > 0);
@@ -992,6 +998,7 @@
memcpy(buf, s->ptr, size);
s->ptr += size;
s->cnt -= size;
+ s->bytes_in += size;
return size;
}
@@ -1002,6 +1009,7 @@
memcpy(buf+1, s->ptr, size);
s->ptr += size;
s->cnt -= size;
+ s->bytes_in += size; /* prot_fill added the 1 already */
return size+1;
}
@@ -1183,6 +1191,7 @@
while (size && (c = prot_getc(s)) != EOF) {
size--;
*p++ = c;
+ s->bytes_in++;
if (c == '\n') break;
}
if (p == buf) return 0;
@@ -1299,6 +1308,7 @@
if (s->cnt > 0) {
--s->cnt;
+ s->bytes_in++;
return *(s->ptr)++;
} else {
return prot_fill(s);
@@ -1310,6 +1320,7 @@
assert(!s->write);
s->cnt++;
+ s->bytes_in--;
*--(s->ptr) = c;
return c;
@@ -1321,6 +1332,7 @@
assert(s->cnt > 0);
*s->ptr++ = c;
+ s->bytes_out++;
if (--s->cnt == 0) {
return prot_flush_internal(s,0);
} else {
Index: cyrus-imapd-2.3.13/lib/prot.h
===================================================================
--- cyrus-imapd-2.3.13.orig/lib/prot.h 2008-11-22 21:48:31.000000000 +1100
+++ cyrus-imapd-2.3.13/lib/prot.h 2008-11-22 22:11:13.000000000 +1100
@@ -105,6 +105,9 @@
time_t timeout_mark;
struct protstream *flushonread;
+ int bytes_in;
+ int bytes_out;
+
/* Events */
prot_readcallback_t *readcallback_proc;
void *readcallback_rock;
@@ -136,9 +139,9 @@
extern int prot_ungetc(int c, struct protstream *s);
extern int prot_putc(int c, struct protstream *s);
-#define prot_getc(s) ((s)->cnt > 0 ? (--(s)->cnt, (int)*(s)->ptr++) : prot_fill(s))
-#define prot_ungetc(c, s) ((s)->cnt++, (*--(s)->ptr = (c)))
-#define prot_putc(c, s) ((*(s)->ptr++ = (c)), --(s)->cnt == 0 ? prot_flush_internal(s,0) : 0)
+#define prot_getc(s) ((s)->cnt > 0 ? (--(s)->cnt, ++(s)->bytes_in, (int)*(s)->ptr++) : prot_fill(s))
+#define prot_ungetc(c, s) ((s)->cnt++, --(s)->bytes_in, (*--(s)->ptr = (c)))
+#define prot_putc(c, s) ((*(s)->ptr++ = (c)), ++(s)->bytes_out, --(s)->cnt == 0 ? prot_flush_internal(s,0) : 0)
/* The following two macros control the blocking nature of
* the protstream.
@@ -168,6 +171,12 @@
/* Set the telemetry logfile for a given protstream */
extern int prot_setlog(struct protstream *s, int fd);
+/* Get traffic counts */
+extern int prot_bytes_in(struct protstream *s);
+extern int prot_bytes_out(struct protstream *s);
+#define prot_bytes_in(s) ((s)->bytes_in)
+#define prot_bytes_out(s) ((s)->bytes_out)
+
/* Set the SASL options for a protstream (requires authentication to
* be complete for the given sasl_conn_t */
extern int prot_setsasl(struct protstream *s, sasl_conn_t *conn);
Index: cyrus-imapd-2.3.13/imap/imapd.c
===================================================================
--- cyrus-imapd-2.3.13.orig/imap/imapd.c 2008-11-22 22:16:13.000000000 +1100
+++ cyrus-imapd-2.3.13/imap/imapd.c 2008-11-22 22:18:58.000000000 +1100
@@ -515,6 +515,8 @@
static void imapd_reset(void)
{
int i;
+ int bytes_in = 0;
+ int bytes_out = 0;
proc_cleanup();
@@ -545,16 +547,18 @@
/* Flush the incoming buffer */
prot_NONBLOCK(imapd_in);
prot_fill(imapd_in);
-
+ bytes_in = prot_bytes_in(imapd_in);
prot_free(imapd_in);
}
if (imapd_out) {
/* Flush the outgoing buffer */
prot_flush(imapd_out);
-
+ bytes_out = prot_bytes_out(imapd_out);
prot_free(imapd_out);
}
+
+ syslog(LOG_NOTICE, "traffic: in=%d out=%d", bytes_in, bytes_out);
imapd_in = imapd_out = NULL;
@@ -848,6 +852,8 @@
void shut_down(int code)
{
int i;
+ int bytes_in = 0;
+ int bytes_out = 0;
proc_cleanup();
@@ -886,19 +892,22 @@
/* Flush the incoming buffer */
prot_NONBLOCK(imapd_in);
prot_fill(imapd_in);
-
+ bytes_in = prot_bytes_in(imapd_in);
prot_free(imapd_in);
}
if (imapd_out) {
/* Flush the outgoing buffer */
prot_flush(imapd_out);
+ bytes_out = prot_bytes_out(imapd_out);
prot_free(imapd_out);
/* one less active connection */
snmp_increment(ACTIVE_CONNECTIONS, -1);
}
+ syslog(LOG_NOTICE, "traffic: in=%d out=%d", bytes_in, bytes_out);
+
if (protin) protgroup_free(protin);
#ifdef HAVE_SSL
Index: cyrus-imapd-2.3.13/imap/pop3d.c
===================================================================
--- cyrus-imapd-2.3.13.orig/imap/pop3d.c 2008-11-22 22:11:23.000000000 +1100
+++ cyrus-imapd-2.3.13/imap/pop3d.c 2008-11-22 22:19:27.000000000 +1100
@@ -313,6 +313,9 @@
static void popd_reset(void)
{
+ int bytes_in = 0;
+ int bytes_out = 0;
+
proc_cleanup();
/* close local mailbox */
@@ -331,14 +334,17 @@
if (popd_in) {
prot_NONBLOCK(popd_in);
prot_fill(popd_in);
-
+ bytes_in = prot_bytes_in(popd_in);
prot_free(popd_in);
}
if (popd_out) {
prot_flush(popd_out);
+ bytes_out = prot_bytes_out(popd_out);
prot_free(popd_out);
}
+
+ syslog(LOG_NOTICE, "traffic: in=%d out=%d", bytes_in, bytes_out);
popd_in = popd_out = NULL;
@@ -590,6 +596,9 @@
*/
void shut_down(int code)
{
+ int bytes_in = 0;
+ int bytes_out = 0;
+
proc_cleanup();
/* close local mailbox */
@@ -616,14 +625,18 @@
if (popd_in) {
prot_NONBLOCK(popd_in);
prot_fill(popd_in);
+ bytes_in = prot_bytes_in(popd_in);
prot_free(popd_in);
}
if (popd_out) {
prot_flush(popd_out);
+ bytes_out = prot_bytes_out(popd_out);
prot_free(popd_out);
}
+ syslog(LOG_NOTICE, "traffic: in=%d out=%d", bytes_in, bytes_out);
+
#ifdef HAVE_SSL
tls_shutdown_serverengine();
#endif
----
Cyrus Home Page: http://cyrusimap.web.cmu.edu/
Cyrus Wiki/FAQ: http://cyrusimap.web.cmu.edu/twiki
List Archives/Info: http://asg.web.cmu.edu/cyrus/mailing-list.html
