Hi all
- Ive just discovered that devel-list strips attachments from messages...
Im proudly present my QUOTA xlator v0.2 'dual fire'. (patch against mainline-2.5 tla636)
"The Quota translator imposes a limit on the underlaying storage allowing to limit disk usage
of subvolumes directly on filesystem calls or by means of informing schedulers on stats mops."
I like to thank avati for his great help on getting me get the inners of glusterFS.
This is the present features (V0.2):
- Check limits and reports limited values from stats mops to upper schedulers (unify et al).
- Cheks limits and reports limited values on statsvfs fops (for "df " command)
- Check limits on writes, reporting ENOSPC on hitting max-size writes
- Checks on unliks computing recovered space.
- Also checks for multiple hardlinks and account on last hardlink unlinked.
Changelog:
Changed unlink tracking scheme "stat before and unlink after" that fired unlink on stat_cbk
on pipeline firing both calls for performance gains.
Changed passing struct stat *buf from posix-storage across call to a private copy on quota xlator
Include docs at /doc/examples/quota.vol
Comments are welcome.
Keep up the Good Work(tm) GlusterFS guys!!!
Regards, Angel
--
------------------------------------------------
Clist UAH (Angel Alvarez)
------------------------------------------------
diff -pruN glusterfs-tla636/configure.ac glusterfs-tla636-quota-xlator-0.1/configure.ac
--- glusterfs-tla636/configure.ac 2008-01-16 17:19:48.290785081 +0100
+++ glusterfs-tla636-quota-xlator-0.1/configure.ac 2008-01-16 22:43:06.896248606 +0100
@@ -64,6 +64,8 @@ AC_CONFIG_FILES([Makefile
xlators/features/fixed-id/src/Makefile
xlators/features/trash/Makefile
xlators/features/trash/src/Makefile
+ xlators/features/quota/Makefile
+ xlators/features/quota/src/Makefile
xlators/encryption/Makefile
xlators/encryption/rot-13/Makefile
xlators/encryption/rot-13/src/Makefile
diff -pruN glusterfs-tla636/doc/examples/quota.vol glusterfs-tla636-quota-xlator-0.1/doc/examples/quota.vol
--- glusterfs-tla636/doc/examples/quota.vol 1970-01-01 01:00:00.000000000 +0100
+++ glusterfs-tla636-quota-xlator-0.1/doc/examples/quota.vol 2008-01-19 22:27:50.234951911 +0100
@@ -0,0 +1,21 @@
+volume brick
+ type storage/posix # POSIX FS translator
+ option directory /home/export # Export this directory
+end-volume
+
+### 'Quota' feature should be added on the server side (as posix volume as subvolume) where it can be used to limit max usage,
+on the underlaying block device.
+volume quota
+ type features/quota
+ subvolumes brick
+ option max-size 50M # Limit underlaying volume to 50MB of space
+ option enforce-level # Limit upper schedulers or limit all oprations (CURRENTLY NOT IMPLEMENTED)
+end-volume
+
+volume server
+ type protocol/server
+ subvolumes quota brick
+ option transport-type tcp/server
+ option auth.ip.brick.allow 192.168.* # Allow access to "brick" volume
+ option auth.ip.quota.allow 192.168.* # Allow access to "quota" volume (exporting 50MB disk space)
+end-volume
diff -pruN glusterfs-tla636/doc/translator-option.txt glusterfs-tla636-quota-xlator-0.1/doc/translator-option.txt
--- glusterfs-tla636/doc/translator-option.txt 2008-01-16 17:19:48.790813576 +0100
+++ glusterfs-tla636-quota-xlator-0.1/doc/translator-option.txt 2008-01-19 22:22:12.215689291 +0100
@@ -151,6 +151,10 @@
option fixed-uid <n> [if not set, not used]
option fixed-gid <n> [if not set, not used]
+# features/quota:
+ option max-size <n> [if not set, not used ]
+ option enforce-level [0|1] (1)
+
# features/filter:
- NO OPTIONS
diff -pruN glusterfs-tla636/xlators/features/Makefile.am glusterfs-tla636-quota-xlator-0.1/xlators/features/Makefile.am
--- glusterfs-tla636/xlators/features/Makefile.am 2008-01-16 17:19:52.291013041 +0100
+++ glusterfs-tla636-quota-xlator-0.1/xlators/features/Makefile.am 2008-01-16 22:24:35.832932716 +0100
@@ -1 +1 @@
-SUBDIRS = filter fixed-id posix-locks trash
+SUBDIRS = filter fixed-id posix-locks trash quota
diff -pruN glusterfs-tla636/xlators/features/quota/Makefile.am glusterfs-tla636-quota-xlator-0.1/xlators/features/quota/Makefile.am
--- glusterfs-tla636/xlators/features/quota/Makefile.am 1970-01-01 01:00:00.000000000 +0100
+++ glusterfs-tla636-quota-xlator-0.1/xlators/features/quota/Makefile.am 2008-01-16 17:19:51.000000000 +0100
@@ -0,0 +1 @@
+SUBDIRS = src
diff -pruN glusterfs-tla636/xlators/features/quota/src/Makefile.am glusterfs-tla636-quota-xlator-0.1/xlators/features/quota/src/Makefile.am
--- glusterfs-tla636/xlators/features/quota/src/Makefile.am 1970-01-01 01:00:00.000000000 +0100
+++ glusterfs-tla636-quota-xlator-0.1/xlators/features/quota/src/Makefile.am 2008-01-16 22:59:55.453723021 +0100
@@ -0,0 +1,11 @@
+
+xlator_PROGRAMS = quota.so
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/quota
+
+quota_so_SOURCES = quota.c
+
+AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
+ -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles
+
+CLEANFILES = *~
+
diff -pruN glusterfs-tla636/xlators/features/quota/src/quota.c glusterfs-tla636-quota-xlator-0.1/xlators/features/quota/src/quota.c
--- glusterfs-tla636/xlators/features/quota/src/quota.c 1970-01-01 01:00:00.000000000 +0100
+++ glusterfs-tla636-quota-xlator-0.1/xlators/features/quota/src/quota.c 2008-01-21 23:32:37.194968044 +0100
@@ -0,0 +1,482 @@
+/*
+ Copyright (c) 2006, 2007 Z RESEARCH, Inc. <http://www.zresearch.com>
+ This file is part of GlusterFS.
+
+ GlusterFS is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published
+ by the Free Software Foundation; either version 3 of the License,
+ or (at your option) any later version.
+
+ GlusterFS is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see
+ <http://www.gnu.org/licenses/>.
+*/
+
+/**
+ * xlators/features/quota :
+ * This translator imposes quota over file operations
+ * mostly based on trace xlator :-).
+ * Derive initially by Angel Alvarez clist@xxxxxx
+ */
+
+#include <time.h>
+#include <errno.h>
+#include "glusterfs.h"
+#include "xlator.h"
+#include "defaults.h"
+
+
+#define ERR_EINVAL_NORETURN(cond) \
+do \
+ { \
+ if ((cond)) \
+ { \
+ gf_log ("ERROR", \
+ GF_LOG_ERROR, \
+ "%s: %s: (%s) is true", \
+ __FILE__, __FUNCTION__, #cond); \
+ } \
+ } while (0)
+
+extern int32_t errno;
+
+#define _FORMAT_WARN(domain, log_level, format, args...) printf ("__DEBUG__" format, ##args);
+
+struct quota_private {
+ int32_t debug_flag;
+ int64_t storage_size;
+ int64_t disk_usage;
+ int64_t free_disk;
+};
+
+void
+quota_add (xlator_t *this,
+ int64_t quantity)
+{
+ struct quota_private *private = NULL;
+
+ if (this) {
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "QUOTA add (%d)",
+ quantity);
+ private = this->private;
+ private->disk_usage += quantity;
+ if ( private->disk_usage > private->storage_size ) {
+ private->free_disk = 0;
+ } else {
+ private->free_disk = private->storage_size - private->disk_usage;
+ }
+ }
+}
+
+void
+quota_sub (xlator_t *this,
+ int64_t quantity)
+{
+ struct quota_private *private = NULL;
+
+ if (this) {
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "QUOTA sub (%d)",
+ quantity);
+ private = this->private;
+ private->disk_usage -= quantity;
+ if ( private->disk_usage < 0 ) {
+ private->free_disk = private->storage_size;
+ } else {
+ private->free_disk = private->storage_size - private->disk_usage;
+ }
+ }
+}
+
+int64_t
+quota_check (xlator_t *this)
+{
+ return ((struct quota_private *)this->private)->free_disk;
+}
+
+void
+quota_alter_statvfs (xlator_t *this,
+ struct statvfs *buf)
+{
+ struct quota_private *private = NULL;
+
+ private = this->private;
+
+ // set filesystem max blocks
+ if ( buf->f_blocks > (private->storage_size / buf->f_frsize) ) buf->f_blocks = (private->storage_size / buf->f_frsize);
+ // set filesystem free blocks for users
+ buf->f_bfree = private->free_disk / buf->f_frsize;
+ // FIXME: set filesystem free blocks for root
+ buf->f_bavail = private->free_disk / buf->f_frsize;
+
+}
+
+static int32_t
+quota_writev_cbk (call_frame_t *frame,
+ void *cookie,
+ xlator_t *this,
+ int32_t op_ret,
+ int32_t op_errno,
+ struct stat *buf)
+{
+ int64_t disk_size = quota_check(this);
+
+ ERR_EINVAL_NORETURN (!this);
+
+ // Record bytes written on success
+ if ( op_ret > 0 ) {
+ quota_add( this, op_ret );
+ }
+ // If quota is over return error
+ if ( disk_size == 0 ) {
+ op_ret = -1;
+ op_errno = ENOSPC;
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "No space available! (*this=%p, op_ret=%d, op_errno=%d)",
+ this, op_ret, op_errno);
+ }
+ STACK_UNWIND (frame, op_ret, op_errno, buf);
+ return 0;
+}
+
+static int32_t
+quota_writev (call_frame_t *frame,
+ xlator_t *this,
+ fd_t *fd,
+ struct iovec *vector,
+ int32_t count,
+ off_t offset)
+{
+ ERR_EINVAL_NORETURN (!this || !fd || !vector || (count < 1));
+
+ STACK_WIND (frame,
+ quota_writev_cbk,
+ FIRST_CHILD(this),
+ FIRST_CHILD(this)->fops->writev,
+ fd, vector, count,offset);
+ return 0;
+}
+
+
+static int32_t
+quota_unlink_cbk (call_frame_t *frame,
+ void *cookie,
+ xlator_t *this,
+ int32_t op_ret,
+ int32_t op_errno)
+{
+ struct stat *mybuf = NULL;
+
+ ERR_EINVAL_NORETURN (!this );
+
+ // get mybuf from frame->local and sanitize it
+ mybuf = frame->local;
+ frame->local = NULL;
+
+ if (op_ret >= 0) {
+ // Unlink Ok, well lets see if there were spare links for the data
+ if ( mybuf->st_nlink == 1 ) quota_sub(this,mybuf->st_size); // no hardlinks left account bytes freed
+ } else {
+ // unlik failed? can not account bytes freed
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "unlink failed! (*this=%p, op_ret=%d, op_errno=%d)",
+ this, op_ret, op_errno);
+ }
+ // free frame->local, so glusterfs hackers dont blame on me!!
+ free(mybuf);
+ // return to caller
+ STACK_UNWIND (frame, op_ret, op_errno);
+ return 0;
+}
+
+static int32_t
+quota_stat_cbk (call_frame_t *frame,
+ void *cookie,
+ xlator_t *this,
+ int32_t op_ret,
+ int32_t op_errno,
+ struct stat *buf)
+{
+ ERR_EINVAL_NORETURN (!this);
+
+ if (frame->local == NULL) {
+
+ // normal unwind
+ STACK_UNWIND (frame, op_ret, op_errno, buf);
+
+ } else {
+
+ // private affairs... i was fired by unlink
+ // copy stat buf onto frame->local (mybuf)
+
+ memcpy(frame->local,buf, sizeof(struct stat));
+
+ // no stack unwinding here
+ }
+ return 0;
+}
+
+static int32_t
+quota_unlink (call_frame_t *frame,
+ xlator_t *this,
+ loc_t *loc)
+{
+ struct stat *mybuf = calloc (1, sizeof (*mybuf));
+
+ ERR_EINVAL_NORETURN (!this || !loc);
+
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ " Unlink request path=%s, inode=%p ",
+ loc->path, loc->inode);
+
+ // store mybuf on frame->local to preserve it
+ frame->local=mybuf;
+ // Fire and forget,stat and unlink
+ STACK_WIND (frame,
+ quota_stat_cbk,
+ FIRST_CHILD(this),
+ FIRST_CHILD(this)->fops->stat,
+ loc);
+ STACK_WIND (frame,
+ quota_unlink_cbk,
+ FIRST_CHILD(this),
+ FIRST_CHILD(this)->fops->unlink,
+ loc);
+ return 0;
+}
+
+static int32_t
+quota_statfs_cbk (call_frame_t *frame,
+ void *cookie,
+ xlator_t *this,
+ int32_t op_ret,
+ int32_t op_errno,
+ struct statvfs *buf)
+{
+
+ ERR_EINVAL_NORETURN (!this);
+
+ if ( op_ret >= 0 ) {
+ quota_alter_statvfs (this, buf);
+ }
+ STACK_UNWIND (frame, op_ret, op_errno, buf);
+ return 0;
+}
+
+static int32_t
+quota_statfs (call_frame_t *frame,
+ xlator_t *this,
+ loc_t *loc)
+{
+ ERR_EINVAL_NORETURN (!this || !loc);
+ STACK_WIND (frame,
+ quota_statfs_cbk,
+ FIRST_CHILD(this),
+ FIRST_CHILD(this)->fops->statfs,
+ loc);
+ return 0;
+}
+
+int32_t
+notify (xlator_t *this,
+ int32_t event,
+ void *data,
+ ...)
+{
+ struct quota_private *private = NULL;
+
+ private = this->private;
+
+ if (!private)
+ return 0;
+
+ switch (event) {
+ case GF_EVENT_CHILD_UP:
+ {
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "Notify Event CHILD_UP (%d)\n", event);
+ default_notify (this, event, data);
+ }
+ break;
+ case GF_EVENT_CHILD_DOWN:
+ {
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "Notify Event CHILD_DOWN (%d)\n", event);
+ default_notify (this, event, data);
+ }
+ break;
+ default:
+ {
+ default_notify (this, event, data);
+ }
+ }
+
+ return 0;
+}
+
+int32_t
+init (xlator_t *this)
+{
+
+void
+gf_log_xlator (xlator_t *this) {
+ int32_t len;
+ char *buf;
+
+ if (!this)
+ return;
+
+ len = dict_serialized_length (this->options);
+ buf = alloca (len);
+ dict_serialize (this->options, buf);
+
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "init name=%s, next=%s, parent=%s ",
+ this->name, this->next->name, this->parent->name);
+}
+
+ dict_t *options = this->options;
+
+ struct quota_private *private = calloc (1, sizeof (*private));
+
+ if (!this)
+ return -1;
+ if (!this->children) {
+ gf_log (this->name,
+ GF_LOG_ERROR,
+ "quota translator requires one subvolume");
+ return -1;
+ }
+
+ if (this->children->next) {
+ gf_log (this->name,
+ GF_LOG_ERROR,
+ "quota translator does not support more than one sub-volume");
+ return -1;
+ }
+ // If parameter "max-size" is set init vars
+ if (dict_get (this->options, "max-size")) {
+ private->storage_size = gf_str_to_long_long (data_to_str (dict_get (this->options,"max-size")));
+ private->disk_usage = 0;
+ private->free_disk = private->storage_size;
+
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "Max filesystem size reported = %s",
+ dict_get (this->options,"max-size"));
+ } else {
+ private->storage_size = 0;
+ }
+
+ //gf_log_set_loglevel (GF_LOG_DEBUG);
+
+
+ gf_log_xlator(this);
+
+ /* Set this translator's inode table pointer to child node's pointer. */
+ this->itable = FIRST_CHILD (this)->itable;
+
+ this->private = (void *)private;
+
+
+ return 0;
+}
+
+
+void
+fini (xlator_t *this)
+{
+ if (!this)
+ return;
+
+
+ /* Free up the dictionary options */
+ dict_destroy (FIRST_CHILD(this)->options);
+
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "quota translator unloaded");
+ return;
+}
+
+struct xlator_fops fops = {
+ .unlink = quota_unlink,
+ .writev = quota_writev,
+ .statfs = quota_statfs,
+};
+
+static int32_t
+quota_stats_cbk (call_frame_t *frame,
+ void *cookie,
+ xlator_t *this,
+ int32_t op_ret,
+ int32_t op_errno,
+ struct xlator_stats *stats)
+{
+ struct quota_private *private = NULL;
+
+
+ private = this->private;
+
+ // Here we control space available to upper modules
+ if (private->storage_size > 0) {
+ // Show original values
+ gf_log (this->name,
+ GF_LOG_DEBUG,
+ "quota_stats_cbk (Limiting storage in stats_cbk)\n");
+ printf ("Pre Limit: %lld Total: %lld Free: %lld Used: %lld \n",
+ private->storage_size,
+ stats->total_disk_size,
+ stats->free_disk,
+ stats->disk_usage);
+ // Alter values for enforce limitation
+ stats->total_disk_size = private->storage_size;
+ stats->free_disk = private->free_disk;
+ stats->disk_usage = private->disk_usage;
+
+ printf ("Post Limit: %lld Total: %lld Free: %lld Used: %lld \n",
+ private->storage_size,
+ stats->total_disk_size,
+ stats->free_disk,
+ stats->disk_usage);
+ }
+ STACK_UNWIND (frame, op_ret, op_errno, stats);
+ return 0;
+}
+
+static int32_t
+quota_stats (call_frame_t *frame,
+ xlator_t *this,
+ int32_t flags)
+{
+ ERR_EINVAL_NORETURN (!this);
+
+ {
+ gf_log (this->name, GF_LOG_DEBUG, "quota_stats (*this=%p, flags=%d\n", this, flags);
+
+ STACK_WIND (frame,
+ quota_stats_cbk,
+ FIRST_CHILD(this),
+ FIRST_CHILD(this)->mops->stats,
+ flags);
+ }
+ return 0;
+}
+
+struct xlator_mops mops = {
+ .stats = quota_stats
+};
