Sanlock is a project that implements a disk-paxos locking algorithm. This is suitable for cluster deployments with shared storage. * src/Makefile.am: Add dlopen plugin for sanlock * src/locking/lock_driver_sanlock.c: Sanlock driver --- po/POTFILES.in | 1 + src/Makefile.am | 12 + src/libvirt_private.syms | 1 + src/locking/lock_driver_sanlock.c | 452 +++++++++++++++++++++++++++++++++++++ 4 files changed, 466 insertions(+), 0 deletions(-) create mode 100644 src/locking/lock_driver_sanlock.c diff --git a/po/POTFILES.in b/po/POTFILES.in index 47f2f20..302b9c0 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -30,6 +30,7 @@ src/interface/netcf_driver.c src/internal.h src/libvirt.c src/locking/lock_manager.c +src/locking/lock_driver_sanlock.c src/lxc/lxc_container.c src/lxc/lxc_conf.c src/lxc/lxc_controller.c diff --git a/src/Makefile.am b/src/Makefile.am index b68a9b4..f56ff17 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -97,6 +97,9 @@ DRIVER_SOURCES = \ locking/lock_driver_nop.h locking/lock_driver_nop.c \ locking/domain_lock.h locking/domain_lock.c +LOCK_DRIVER_SANLOCK_SOURCES = \ + locking/lock_driver_sanlock.c + # XML configuration format handling sources # Domain driver generic impl APIs @@ -1148,6 +1151,15 @@ libvirt_qemu_la_CFLAGS = $(AM_CFLAGS) libvirt_qemu_la_LIBADD = libvirt.la $(CYGWIN_EXTRA_LIBADD) EXTRA_DIST += $(LIBVIRT_QEMU_SYMBOL_FILE) + +lockdriverdir = $(libdir)/libvirt/lock-driver +lockdriver_LTLIBRARIES = sanlock.la + +sanlock_la_SOURCES = $(LOCK_DRIVER_SANLOCK_SOURCES) +sanlock_la_CFLAGS = $(AM_CLFAGS) +sanlock_la_LDFLAGS = -no-version -module +sanlock_la_LIBADD = -lsanlock + libexec_PROGRAMS = if WITH_STORAGE_DISK diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 8005f20..b2f11b8 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -585,6 +585,7 @@ virVMOperationTypeToString; # memory.h virAlloc; virAllocN; +virAllocVar; virExpandN; virFree; virReallocN; diff --git a/src/locking/lock_driver_sanlock.c b/src/locking/lock_driver_sanlock.c new file mode 100644 index 0000000..90afe18 --- /dev/null +++ b/src/locking/lock_driver_sanlock.c @@ -0,0 +1,452 @@ +/* + * lock_driver_sanlock.c: A lock driver for Sanlock + * + * Copyright (C) 2010-2011 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + */ + +#include <config.h> + +#include <stdlib.h> +#include <stdint.h> +#include <unistd.h> +#include <string.h> +#include <stdio.h> +#include <errno.h> +#include <sys/types.h> + +#include <sanlock.h> +#include <sanlock_resource.h> + +#include "lock_driver.h" +#include "logging.h" +#include "virterror_internal.h" +#include "memory.h" +#include "util.h" +#include "files.h" + +#define VIR_FROM_THIS VIR_FROM_LOCKING + +#define virLockError(code, ...) \ + virReportErrorHelper(NULL, VIR_FROM_THIS, code, __FILE__, \ + __FUNCTION__, __LINE__, __VA_ARGS__) + +struct snlk_con { + char vm_name[SANLK_NAME_LEN]; + char vm_uuid[VIR_UUID_BUFLEN]; + unsigned int vm_id; + unsigned int vm_pid; + unsigned int flags; + int sock; + int res_count; + struct sanlk_resource *res_args[SANLK_MAX_RESOURCES]; +}; + +/* + * sanlock plugin for the libvirt virLockManager API + */ + +static int drv_snlk_init(unsigned int version ATTRIBUTE_UNUSED, + unsigned int flags) +{ + virCheckFlags(VIR_LOCK_MANAGER_MODE_CONTENT, -1); + return 0; +} + +static int drv_snlk_deinit(void) +{ + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unloading sanlock plugin is forbidden")); + return -1; +} + +static int drv_snlk_new(virLockManagerPtr man, + unsigned int type, + size_t nparams, + virLockManagerParamPtr params, + unsigned int flags) +{ + virLockManagerParamPtr param; + struct snlk_con *con; + int i; + + virCheckFlags(VIR_LOCK_MANAGER_MODE_CONTENT, -1); + + if (type != VIR_LOCK_MANAGER_OBJECT_TYPE_DOMAIN) { + virLockError(VIR_ERR_INTERNAL_ERROR, + _("Unsupported object type %d"), type); + return -1; + } + + if (VIR_ALLOC(con) < 0) { + virReportOOMError(); + return -1; + } + + con->flags = flags; + con->sock = -1; + + for (i = 0; i < nparams; i++) { + param = ¶ms[i]; + + if (STREQ(param->key, "uuid")) { + memcpy(con->vm_uuid, param->value.uuid, 16); + } else if (STREQ(param->key, "name")) { + if (!virStrcpy(con->vm_name, param->value.str, SANLK_NAME_LEN)) { + virLockError(VIR_ERR_INTERNAL_ERROR, + _("Domain name '%s' exceeded %d characters"), + param->value.str, SANLK_NAME_LEN); + goto error; + } + } else if (STREQ(param->key, "pid")) { + con->vm_pid = param->value.ui; + } else if (STREQ(param->key, "id")) { + con->vm_id = param->value.ui; + } + } + + man->privateData = con; + return 0; + +error: + VIR_FREE(con); + return -1; +} + +static void drv_snlk_free(virLockManagerPtr man) +{ + struct snlk_con *con = man->privateData; + + DEBUG("man=%p sock=%d", man, con->sock); +#if 0 + /* We do *not* want to close the socket here. We need the + * socket to keep alive other sanlock will fence the + * process. The socket will be explicitly closed before + * free, in the release_object method, if neccessary. + */ + VIR_FORCE_CLOSE(con->sock); +#endif + VIR_FREE(con); + man->privateData = NULL; +} + +static int add_con_resource(struct snlk_con *con, + const char *name, + size_t nparams, + virLockManagerParamPtr params) +{ + virLockManagerParamPtr param; + struct sanlk_resource *res; + int i; + + if (VIR_ALLOC_VAR(res, struct sanlk_disk, 1) < 0) { + virReportOOMError(); + return -1; + } + + res->num_disks = 1; + if (!virStrcpy(res->name, name, SANLK_NAME_LEN)) { + virLockError(VIR_ERR_INTERNAL_ERROR, + _("Resource name '%s' exceeds %d characters"), + name, SANLK_NAME_LEN); + goto error; + } + + for (i = 0; i < nparams; i++) { + param = ¶ms[i]; + + if (STREQ(param->key, "path")) { + if (!virStrcpy(res->disks[0].path, param->value.str, SANLK_PATH_LEN)) { + virLockError(VIR_ERR_INTERNAL_ERROR, + _("Lease path '%s' exceeds %d characters"), + param->value.str, SANLK_PATH_LEN); + goto error; + } + } else if (STREQ(param->key, "offset")) { + res->disks[0].offset = param->value.ul; + } + } + + con->res_args[con->res_count] = res; + con->res_count++; + return 0; + +error: + VIR_FREE(res); + return -1; +} + +static int drv_snlk_add_resource(virLockManagerPtr man, + unsigned int type, + const char *name, + size_t nparams, + virLockManagerParamPtr params, + unsigned int flags ATTRIBUTE_UNUSED) +{ + struct snlk_con *con = man->privateData; + /* must be called before acquire_object */ + if (con->sock != -1) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot add resources to an existing lock")); + return -1; + } + + if (con->res_count == SANLK_MAX_RESOURCES) { + virLockError(VIR_ERR_INTERNAL_ERROR, + _("Too many resources %d for object"), + SANLK_MAX_RESOURCES); + return -1; + } + + if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE) + return 0; + + if (add_con_resource(con, name, nparams, params) < 0) + return -1; + + return 0; +} + +static int drv_snlk_acquire_object(virLockManagerPtr man, + const char *state, + unsigned int flags ATTRIBUTE_UNUSED) +{ + struct snlk_con *con = man->privateData; + struct sanlk_options *opt = NULL; + int i, rv, sock; + int pid = getpid(); + + /* acquire_object can be called only once */ + if (con->sock != -1) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Object lock is already held")); + return -1; + } + + if (con->vm_pid != pid) { + virLockError(VIR_ERR_INTERNAL_ERROR, + _("Object lock attempt from pid %d, expected %d"), + pid, con->vm_pid); + return -1; + } + + if (VIR_ALLOC_VAR(opt, char, state ? strlen(state) : 0) < 0) { + virReportOOMError(); + return -1; + } + + if (!virStrcpy(opt->owner_name, con->vm_name, SANLK_NAME_LEN)) { + virLockError(VIR_ERR_INTERNAL_ERROR, + _("Domain name '%s' exceeded %d characters"), + con->vm_name, SANLK_NAME_LEN); + goto error; + } + + if (state) { + opt->flags = SANLK_FLG_INCOMING; + opt->len = strlen(state); + strcpy(opt->str, state); + } + + VIR_DEBUG0("Register sanlock"); + sock = sanlock_register(); + if (sock < 0) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to open socket to sanlock daemon")); + goto error; + } + VIR_DEBUG("Acquiring object %u", con->res_count); + rv = sanlock_acquire(sock, -1, con->res_count, con->res_args, opt); + VIR_DEBUG("Acquire result %d", rv); + if (rv < 0) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Failed to acquire lock")); + goto error; + } + VIR_FREE(opt); + + for (i = 0; i < con->res_count; i++) + VIR_FREE(con->res_args[i]); + + con->sock = sock; + + return 0; + +error: + VIR_FORCE_CLOSE(sock); + VIR_FREE(opt); + return -1; +} + +static int drv_snlk_attach_object(virLockManagerPtr man ATTRIBUTE_UNUSED, + unsigned int flags ATTRIBUTE_UNUSED) +{ + return 0; +} + +static int drv_snlk_detach_object(virLockManagerPtr man ATTRIBUTE_UNUSED, + unsigned int flags ATTRIBUTE_UNUSED) +{ + return 0; +} + +static int drv_snlk_release_object(virLockManagerPtr man ATTRIBUTE_UNUSED, + unsigned int flags ATTRIBUTE_UNUSED) +{ + struct snlk_con *con = man->privateData; + + if (con->sock == -1) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot release object that is not locked")); + return -1; + } + VIR_FORCE_CLOSE(con->sock); + return 0; +} + +static int drv_snlk_get_state(virLockManagerPtr man ATTRIBUTE_UNUSED, + char **state, + unsigned int flags ATTRIBUTE_UNUSED) +{ + *state = NULL; + + return 0; +} + + +static int drv_snlk_acquire_resource(virLockManagerPtr man, + unsigned int type, + const char *name, + size_t nparams, + virLockManagerParamPtr params, + unsigned int flags ATTRIBUTE_UNUSED) +{ + struct snlk_con *con = man->privateData; + struct sanlk_options opt; + int rv; + + if (con->sock != -1) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot acquire resource on unlocked object")); + return -1; + } + + if (!con->vm_pid) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot acquire resource on unlocked object")); + return -1; + } + + if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE) + return 0; + + if (add_con_resource(con, name, nparams, params) < 0) + return -1; + + /* Setting REACQUIRE tells sanlock that if con->vm_pid previously held + and released the resource, we need to ensure no other host has + acquired a lease on it in the mean time. If this is a new resource + that the pid hasn't held before, then REACQUIRE will have no effect + since sanlock will have no memory of a previous version. */ + + memset(&opt, 0, sizeof(struct sanlk_options)); + if (!virStrcpy(opt.owner_name, con->vm_name, SANLK_NAME_LEN)) { + virLockError(VIR_ERR_INTERNAL_ERROR, + _("Domain name '%s' exceeds %d characters"), + con->vm_name, SANLK_NAME_LEN); + return -1; + } + opt.flags = SANLK_FLG_REACQUIRE; + opt.len = 0; + + rv = sanlock_acquire(-1, con->vm_pid, con->res_count, con->res_args, &opt); + + VIR_FREE(con->res_args[0]); + + if (rv < 0) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to acquire resource")); + return -1; + } + + return 0; +} + +static int drv_snlk_release_resource(virLockManagerPtr man, + unsigned int type, + const char *name, + size_t nparams, + virLockManagerParamPtr params, + unsigned int flags ATTRIBUTE_UNUSED) +{ + struct snlk_con *con = man->privateData; + int rv; + + if (con->sock != -1) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot acquire resource on unlocked object")); + return -1; + } + + if (!con->vm_pid) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Cannot acquire resource on unlocked object")); + return -1; + } + + if (type != VIR_LOCK_MANAGER_RESOURCE_TYPE_LEASE) + return 0; + + if (add_con_resource(con, name, nparams, params) < 0) + return -1; + + rv = sanlock_release(-1, con->vm_pid, con->res_count, con->res_args); + + VIR_FREE(con->res_args[0]); + + if (rv < 0) { + virLockError(VIR_ERR_INTERNAL_ERROR, "%s", + _("Unable to release resource")); + return -1; + } + + return 0; +} + +virLockDriver virLockDriverImpl = +{ + .version = VIR_LOCK_MANAGER_VERSION, + .flags = VIR_LOCK_MANAGER_MODE_CONTENT, + + .drvInit = drv_snlk_init, + .drvDeinit = drv_snlk_deinit, + + .drvNew = drv_snlk_new, + .drvFree = drv_snlk_free, + + .drvAddResource = drv_snlk_add_resource, + + .drvAcquireObject = drv_snlk_acquire_object, + .drvAttachObject = drv_snlk_attach_object, + .drvDetachObject = drv_snlk_detach_object, + .drvReleaseObject = drv_snlk_release_object, + + .drvGetState = drv_snlk_get_state, + + .drvAcquireResource = drv_snlk_acquire_resource, + .drvReleaseResource = drv_snlk_release_resource, +}; -- 1.7.3.4 -- libvir-list mailing list libvir-list@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/libvir-list