From: "Fabio M. Di Nitto" <fdinitto@xxxxxxxxxx>
this change should address the last issues, reduce again size of the onwire
and make registration atomic and safe.
Similar approach is used for unregister.
qdevice name update/reconfigure is only partially implemented, but
some bits are stubs.
Signed-off-by: Fabio M. Di Nitto <fdinitto@xxxxxxxxxx>
---
exec/votequorum.c | 290 +++++++++++++++++++++++++++++++++++++++++++++--------
1 files changed, 250 insertions(+), 40 deletions(-)
diff --git a/exec/votequorum.c b/exec/votequorum.c
index 3765a69..a5c9883 100644
--- a/exec/votequorum.c
+++ b/exec/votequorum.c
@@ -64,10 +64,12 @@ static struct corosync_api_v1 *corosync_api;
#define DEFAULT_QDEVICE_TIMEOUT 10000
+static char qdevice_name[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
static struct cluster_node *qdevice = NULL;
static unsigned int qdevice_timeout = DEFAULT_QDEVICE_TIMEOUT;
static uint8_t qdevice_can_operate = 1;
static uint8_t qdevice_is_registered = 0;
+static void *qdevice_reg_conn = NULL;
static uint8_t two_node = 0;
@@ -94,7 +96,6 @@ struct req_exec_quorum_nodeinfo {
uint32_t votes;
uint32_t expected_votes;
uint32_t flags;
- char qdevice_name[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
} __attribute__((packed));
struct req_exec_quorum_reconfigure {
@@ -107,6 +108,18 @@ struct req_exec_quorum_reconfigure {
uint8_t _pad2;
} __attribute__((packed));
+struct req_exec_quorum_qdevice_reg {
+ struct qb_ipc_request_header header __attribute__((aligned(8)));
+ uint32_t operation;
+ char qdevice_name[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
+} __attribute__((packed));
+
+struct req_exec_quorum_qdevice_reconfigure {
+ struct qb_ipc_request_header header __attribute__((aligned(8)));
+ char oldname[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
+ char newname[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
+} __attribute__((packed));
+
/*
* votequorum_exec onwire version (via totem)
*/
@@ -117,8 +130,10 @@ struct req_exec_quorum_reconfigure {
* votequorum_exec onwire messages (via totem)
*/
-#define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO 0
-#define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE 1
+#define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO 0
+#define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE 1
+#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG 2
+#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE 3
static void votequorum_exec_send_expectedvotes_notification(void);
static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context);
@@ -129,6 +144,12 @@ static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context
static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid, uint32_t value);
/*
+ * used by req_exec_quorum_qdevice_reg
+ */
+#define VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER 0
+#define VOTEQUORUM_QDEVICE_OPERATION_REGISTER 1
+
+/*
* votequorum internal node status/view
*/
@@ -153,7 +174,6 @@ struct cluster_node {
uint32_t votes;
uint32_t expected_votes;
uint32_t flags;
- char qdevice_name[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
struct list_head list;
};
@@ -227,6 +247,16 @@ static void message_handler_req_exec_votequorum_reconfigure (
unsigned int nodeid);
static void exec_votequorum_reconfigure_endian_convert (void *message);
+static void message_handler_req_exec_votequorum_qdevice_reg (
+ const void *message,
+ unsigned int nodeid);
+static void exec_votequorum_qdevice_reg_endian_convert (void *message);
+
+static void message_handler_req_exec_votequorum_qdevice_reconfigure (
+ const void *message,
+ unsigned int nodeid);
+static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message);
+
static struct corosync_exec_handler votequorum_exec_engine[] =
{
{ /* 0 */
@@ -237,6 +267,14 @@ static struct corosync_exec_handler votequorum_exec_engine[] =
.exec_handler_fn = message_handler_req_exec_votequorum_reconfigure,
.exec_endian_convert_fn = exec_votequorum_reconfigure_endian_convert
},
+ { /* 2 */
+ .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reg,
+ .exec_endian_convert_fn = exec_votequorum_qdevice_reg_endian_convert
+ },
+ { /* 3 */
+ .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reconfigure,
+ .exec_endian_convert_fn = exec_votequorum_qdevice_reconfigure_endian_convert
+ },
};
/*
@@ -1144,11 +1182,6 @@ static int votequorum_exec_send_nodeinfo(uint32_t nodeid)
req_exec_quorum_nodeinfo.votes = node->votes;
req_exec_quorum_nodeinfo.expected_votes = node->expected_votes;
req_exec_quorum_nodeinfo.flags = node->flags;
- if ((nodeid != NODEID_QDEVICE) && (qdevice_is_registered)) {
- strcpy(req_exec_quorum_nodeinfo.qdevice_name, qdevice->qdevice_name);
- } else {
- memset(req_exec_quorum_nodeinfo.qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
- }
req_exec_quorum_nodeinfo.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO);
req_exec_quorum_nodeinfo.header.size = sizeof(req_exec_quorum_nodeinfo);
@@ -1162,6 +1195,50 @@ static int votequorum_exec_send_nodeinfo(uint32_t nodeid)
return ret;
}
+static int votequorum_exec_send_qdevice_reconfigure(const char *oldname, const char *newname)
+{
+ struct req_exec_quorum_qdevice_reconfigure req_exec_quorum_qdevice_reconfigure;
+ struct iovec iov[1];
+ int ret;
+
+ ENTER();
+
+ req_exec_quorum_qdevice_reconfigure.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE);
+ req_exec_quorum_qdevice_reconfigure.header.size = sizeof(req_exec_quorum_qdevice_reconfigure);
+ strcpy(req_exec_quorum_qdevice_reconfigure.oldname, oldname);
+ strcpy(req_exec_quorum_qdevice_reconfigure.newname, newname);
+
+ iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reconfigure;
+ iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reconfigure);
+
+ ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
+
+ LEAVE();
+ return ret;
+}
+
+static int votequorum_exec_send_qdevice_reg(uint32_t operation, const char *qdevice_name_req)
+{
+ struct req_exec_quorum_qdevice_reg req_exec_quorum_qdevice_reg;
+ struct iovec iov[1];
+ int ret;
+
+ ENTER();
+
+ req_exec_quorum_qdevice_reg.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG);
+ req_exec_quorum_qdevice_reg.header.size = sizeof(req_exec_quorum_qdevice_reg);
+ req_exec_quorum_qdevice_reg.operation = operation;
+ strcpy(req_exec_quorum_qdevice_reg.qdevice_name, qdevice_name_req);
+
+ iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reg;
+ iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reg);
+
+ ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
+
+ LEAVE();
+ return ret;
+}
+
static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context)
{
struct res_lib_votequorum_notification *res_lib_votequorum_notification;
@@ -1251,6 +1328,136 @@ static void votequorum_exec_send_expectedvotes_notification(void)
LEAVE();
}
+static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message)
+{
+ ENTER();
+
+ LEAVE();
+}
+
+static void message_handler_req_exec_votequorum_qdevice_reconfigure (
+ const void *message,
+ unsigned int nodeid)
+{
+ const struct req_exec_quorum_qdevice_reconfigure *req_exec_quorum_qdevice_reconfigure = message;
+
+ ENTER();
+
+ log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice name change req from node %u [from: %s to: %s]",
+ nodeid,
+ req_exec_quorum_qdevice_reconfigure->oldname,
+ req_exec_quorum_qdevice_reconfigure->newname);
+
+ if (!strcmp(req_exec_quorum_qdevice_reconfigure->oldname, qdevice_name)) {
+ log_printf(LOGSYS_LEVEL_DEBUG, "Allowing qdevice rename");
+ memset(qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
+ strcpy(qdevice_name, req_exec_quorum_qdevice_reconfigure->newname);
+ /*
+ * TODO: notify qdevices about name change?
+ * this is not relevant for now and can wait later on since
+ * qdevices are local only and libvotequorum is not final
+ */
+ }
+
+ LEAVE();
+}
+
+static void exec_votequorum_qdevice_reg_endian_convert (void *message)
+{
+ struct req_exec_quorum_qdevice_reg *req_exec_quorum_qdevice_reg = message;
+
+ ENTER();
+
+ req_exec_quorum_qdevice_reg->operation = swab32(req_exec_quorum_qdevice_reg->operation);
+
+ LEAVE();
+}
+
+static void message_handler_req_exec_votequorum_qdevice_reg (
+ const void *message,
+ unsigned int nodeid)
+{
+ const struct req_exec_quorum_qdevice_reg *req_exec_quorum_qdevice_reg = message;
+ struct res_lib_votequorum_status res_lib_votequorum_status;
+ int wipe_qdevice_name = 1;
+ struct cluster_node *node = NULL;
+ struct list_head *tmp;
+ cs_error_t error = CS_OK;
+
+ ENTER();
+
+ log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice op %u req from node %u [%s]",
+ req_exec_quorum_qdevice_reg->operation,
+ nodeid, req_exec_quorum_qdevice_reg->qdevice_name);
+
+ switch(req_exec_quorum_qdevice_reg->operation)
+ {
+ case VOTEQUORUM_QDEVICE_OPERATION_REGISTER:
+ if (nodeid != us->node_id) {
+ if (!strlen(qdevice_name)) {
+ log_printf(LOGSYS_LEVEL_DEBUG, "Remote qdevice name recorded");
+ strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name);
+ }
+ LEAVE();
+ return;
+ }
+
+ /*
+ * this should NEVER happen
+ */
+ if (!qdevice_reg_conn) {
+ log_printf(LOGSYS_LEVEL_WARNING, "Unable to determine origin of the qdevice register call!");
+ LEAVE();
+ return;
+ }
+
+ /*
+ * registering our own device in this case
+ */
+ if (!strlen(qdevice_name)) {
+ strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name);
+ }
+
+ /*
+ * check if it is our device or something else
+ */
+ if ((!strncmp(req_exec_quorum_qdevice_reg->qdevice_name,
+ qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN))) {
+ qdevice_is_registered = 1;
+ us->flags |= NODE_FLAGS_QDEVICE;
+ votequorum_exec_send_nodeinfo(NODEID_QDEVICE);
+ votequorum_exec_send_nodeinfo(us->node_id);
+ } else {
+ log_printf(LOGSYS_LEVEL_WARNING,
+ "A new qdevice with different name (new: %s old: %s) is trying to register!",
+ req_exec_quorum_qdevice_reg->qdevice_name, qdevice_name);
+ error = CS_ERR_EXIST;
+ }
+
+ res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status);
+ res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS;
+ res_lib_votequorum_status.header.error = error;
+ corosync_api->ipc_response_send(qdevice_reg_conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status));
+ qdevice_reg_conn = NULL;
+ break;
+ case VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER:
+ list_iterate(tmp, &cluster_members_list) {
+ node = list_entry(tmp, struct cluster_node, list);
+ if ((node->state == NODESTATE_MEMBER) &&
+ (node->flags & NODE_FLAGS_QDEVICE)) {
+ wipe_qdevice_name = 0;
+ }
+ }
+
+ if (wipe_qdevice_name) {
+ memset(qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
+ }
+
+ break;
+ }
+ LEAVE();
+}
+
static void exec_votequorum_nodeinfo_endian_convert (void *message)
{
struct req_exec_quorum_nodeinfo *nodeinfo = message;
@@ -1317,19 +1524,6 @@ static void message_handler_req_exec_votequorum_nodeinfo (
if (nodeid != NODEID_QDEVICE) {
node->votes = req_exec_quorum_nodeinfo->votes;
-
- memset(node->qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
- if (node->flags & NODE_FLAGS_QDEVICE) {
- int state = 0;
-
- if (node->flags & NODE_FLAGS_QDEVICE_STATE) {
- state = 1;
- }
-
- strcpy(node->qdevice_name, req_exec_quorum_nodeinfo->qdevice_name);
- log_printf(LOGSYS_LEVEL_DEBUG, "Got qdevice data from node[%u]: state %u name %s",
- node->node_id, state, node->qdevice_name);
- }
} else {
if ((!cluster_is_quorate) &&
(req_exec_quorum_nodeinfo->flags & NODE_FLAGS_QUORATE)) {
@@ -1500,7 +1694,7 @@ static char *votequorum_exec_init_fn (struct corosync_api_v1 *api)
}
qdevice->state = NODESTATE_DEAD;
qdevice->votes = 0;
- memset(qdevice->qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
+ memset(qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
votequorum_readconfig_dynamic();
recalculate_quorum(0, 0);
@@ -1580,6 +1774,10 @@ static void votequorum_confchg_fn (
quorum_members_entries = member_list_entries;
votequorum_exec_send_nodeinfo(us->node_id);
votequorum_exec_send_nodeinfo(NODEID_QDEVICE);
+ if (strlen(qdevice_name)) {
+ votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER,
+ qdevice_name);
+ }
}
memcpy(&quorum_ringid, ring_id, sizeof(*ring_id));
@@ -1677,7 +1875,7 @@ static void qdevice_timer_fn(void *arg)
qdevice->state = NODESTATE_DEAD;
us->flags &= ~NODE_FLAGS_QDEVICE_STATE;
- log_printf(LOGSYS_LEVEL_INFO, "lost contact with quorum device %s", qdevice->qdevice_name);
+ log_printf(LOGSYS_LEVEL_INFO, "lost contact with quorum device %s", qdevice_name);
votequorum_exec_send_nodeinfo(us->node_id);
qdevice_timer_set = 0;
@@ -1932,21 +2130,33 @@ static void message_handler_req_lib_votequorum_qdevice_register (void *conn,
if (qdevice_is_registered) {
if ((!strncmp(req_lib_votequorum_qdevice_register->name,
- qdevice->qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN))) {
+ qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN))) {
goto out;
} else {
log_printf(LOGSYS_LEVEL_WARNING,
"A new qdevice with different name (new: %s old: %s) is trying to re-register!",
- req_lib_votequorum_qdevice_register->name, qdevice->qdevice_name);
+ req_lib_votequorum_qdevice_register->name, qdevice_name);
error = CS_ERR_EXIST;
goto out;
}
} else {
- qdevice_is_registered = 1;
- strcpy(qdevice->qdevice_name, req_lib_votequorum_qdevice_register->name);
- us->flags |= NODE_FLAGS_QDEVICE;
- votequorum_exec_send_nodeinfo(NODEID_QDEVICE);
- votequorum_exec_send_nodeinfo(us->node_id);
+ if (qdevice_reg_conn != NULL) {
+ log_printf(LOGSYS_LEVEL_WARNING,
+ "Registration request already in progress");
+ error = CS_ERR_TRY_AGAIN;
+ goto out;
+ }
+ qdevice_reg_conn = conn;
+ if (votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER,
+ req_lib_votequorum_qdevice_register->name) != 0) {
+ log_printf(LOGSYS_LEVEL_WARNING,
+ "Unable to send qdevice registration request to cluster");
+ error = CS_ERR_TRY_AGAIN;
+ qdevice_reg_conn = NULL;
+ } else {
+ LEAVE();
+ return;
+ }
}
out:
@@ -1969,7 +2179,7 @@ static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn,
ENTER();
if (qdevice_is_registered) {
- if (strncmp(req_lib_votequorum_qdevice_unregister->name, qdevice->qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
+ if (strncmp(req_lib_votequorum_qdevice_unregister->name, qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
error = CS_ERR_INVALID_PARAM;
goto out;
}
@@ -1981,8 +2191,9 @@ static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn,
us->flags &= ~NODE_FLAGS_QDEVICE;
us->flags &= ~NODE_FLAGS_QDEVICE_STATE;
qdevice->state = NODESTATE_DEAD;
- memset(qdevice->qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
votequorum_exec_send_nodeinfo(us->node_id);
+ votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER,
+ req_lib_votequorum_qdevice_unregister->name);
} else {
error = CS_ERR_NOT_EXIST;
}
@@ -2006,13 +2217,12 @@ static void message_handler_req_lib_votequorum_qdevice_update (void *conn,
ENTER();
if (qdevice_is_registered) {
- if (strncmp(req_lib_votequorum_qdevice_update->oldname, qdevice->qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
+ if (strncmp(req_lib_votequorum_qdevice_update->oldname, qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
error = CS_ERR_INVALID_PARAM;
goto out;
}
- memset(qdevice->qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
- strcpy(qdevice->qdevice_name, req_lib_votequorum_qdevice_update->newname);
- votequorum_exec_send_nodeinfo(us->node_id);
+ votequorum_exec_send_qdevice_reconfigure(req_lib_votequorum_qdevice_update->oldname,
+ req_lib_votequorum_qdevice_update->newname);
} else {
error = CS_ERR_NOT_EXIST;
}
@@ -2041,7 +2251,7 @@ static void message_handler_req_lib_votequorum_qdevice_poll (void *conn,
}
if (qdevice_is_registered) {
- if (strncmp(req_lib_votequorum_qdevice_poll->name, qdevice->qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
+ if (strncmp(req_lib_votequorum_qdevice_poll->name, qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
error = CS_ERR_INVALID_PARAM;
goto out;
}
@@ -2108,7 +2318,7 @@ static void message_handler_req_lib_votequorum_qdevice_getinfo (void *conn,
} else {
res_lib_votequorum_qdevice_getinfo.state = 0;
}
- strcpy(res_lib_votequorum_qdevice_getinfo.name, node->qdevice_name);
+ strcpy(res_lib_votequorum_qdevice_getinfo.name, qdevice_name);
} else {
error = CS_ERR_NOT_EXIST;
}
@@ -2121,7 +2331,7 @@ static void message_handler_req_lib_votequorum_qdevice_getinfo (void *conn,
} else {
res_lib_votequorum_qdevice_getinfo.state = 0;
}
- strcpy(res_lib_votequorum_qdevice_getinfo.name, qdevice->qdevice_name);
+ strcpy(res_lib_votequorum_qdevice_getinfo.name, qdevice_name);
} else {
error = CS_ERR_NOT_EXIST;
}
--
1.7.7.6
_______________________________________________
discuss mailing list
discuss@xxxxxxxxxxxx
http://lists.corosync.org/mailman/listinfo/discuss
