Reviewed-by: Steven Dake <sdake@xxxxxxxxxx>
On 02/24/2012 07:17 AM, Fabio M. Di Nitto wrote:
> From: "Fabio M. Di Nitto" <fdinitto@xxxxxxxxxx>
>
> this change should address the last issues, reduce again size of the onwire
> and make registration atomic and safe.
>
> Similar approach is used for unregister.
>
> qdevice name update/reconfigure is only partially implemented, but
> some bits are stubs.
>
> Signed-off-by: Fabio M. Di Nitto <fdinitto@xxxxxxxxxx>
> ---
> exec/votequorum.c | 290 +++++++++++++++++++++++++++++++++++++++++++++--------
> 1 files changed, 250 insertions(+), 40 deletions(-)
>
> diff --git a/exec/votequorum.c b/exec/votequorum.c
> index 3765a69..a5c9883 100644
> --- a/exec/votequorum.c
> +++ b/exec/votequorum.c
> @@ -64,10 +64,12 @@ static struct corosync_api_v1 *corosync_api;
>
> #define DEFAULT_QDEVICE_TIMEOUT 10000
>
> +static char qdevice_name[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
Is this a filename? IF so, PATH_MAX would be better choice for define
> static struct cluster_node *qdevice = NULL;
> static unsigned int qdevice_timeout = DEFAULT_QDEVICE_TIMEOUT;
> static uint8_t qdevice_can_operate = 1;
> static uint8_t qdevice_is_registered = 0;
> +static void *qdevice_reg_conn = NULL;
>
> static uint8_t two_node = 0;
>
> @@ -94,7 +96,6 @@ struct req_exec_quorum_nodeinfo {
> uint32_t votes;
> uint32_t expected_votes;
> uint32_t flags;
> - char qdevice_name[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
> } __attribute__((packed));
>
> struct req_exec_quorum_reconfigure {
> @@ -107,6 +108,18 @@ struct req_exec_quorum_reconfigure {
> uint8_t _pad2;
> } __attribute__((packed));
>
> +struct req_exec_quorum_qdevice_reg {
> + struct qb_ipc_request_header header __attribute__((aligned(8)));
> + uint32_t operation;
> + char qdevice_name[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
> +} __attribute__((packed));
> +
> +struct req_exec_quorum_qdevice_reconfigure {
> + struct qb_ipc_request_header header __attribute__((aligned(8)));
> + char oldname[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
> + char newname[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
> +} __attribute__((packed));
> +
> /*
> * votequorum_exec onwire version (via totem)
> */
> @@ -117,8 +130,10 @@ struct req_exec_quorum_reconfigure {
> * votequorum_exec onwire messages (via totem)
> */
>
> -#define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO 0
> -#define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE 1
> +#define MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO 0
> +#define MESSAGE_REQ_EXEC_VOTEQUORUM_RECONFIGURE 1
> +#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG 2
> +#define MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE 3
>
> static void votequorum_exec_send_expectedvotes_notification(void);
> static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context);
> @@ -129,6 +144,12 @@ static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context
> static int votequorum_exec_send_reconfigure(uint8_t param, unsigned int nodeid, uint32_t value);
>
> /*
> + * used by req_exec_quorum_qdevice_reg
> + */
> +#define VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER 0
> +#define VOTEQUORUM_QDEVICE_OPERATION_REGISTER 1
> +
> +/*
> * votequorum internal node status/view
> */
>
> @@ -153,7 +174,6 @@ struct cluster_node {
> uint32_t votes;
> uint32_t expected_votes;
> uint32_t flags;
> - char qdevice_name[VOTEQUORUM_MAX_QDEVICE_NAME_LEN];
> struct list_head list;
> };
>
> @@ -227,6 +247,16 @@ static void message_handler_req_exec_votequorum_reconfigure (
> unsigned int nodeid);
> static void exec_votequorum_reconfigure_endian_convert (void *message);
>
> +static void message_handler_req_exec_votequorum_qdevice_reg (
> + const void *message,
> + unsigned int nodeid);
> +static void exec_votequorum_qdevice_reg_endian_convert (void *message);
> +
> +static void message_handler_req_exec_votequorum_qdevice_reconfigure (
> + const void *message,
> + unsigned int nodeid);
> +static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message);
> +
> static struct corosync_exec_handler votequorum_exec_engine[] =
> {
> { /* 0 */
> @@ -237,6 +267,14 @@ static struct corosync_exec_handler votequorum_exec_engine[] =
> .exec_handler_fn = message_handler_req_exec_votequorum_reconfigure,
> .exec_endian_convert_fn = exec_votequorum_reconfigure_endian_convert
> },
> + { /* 2 */
> + .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reg,
> + .exec_endian_convert_fn = exec_votequorum_qdevice_reg_endian_convert
> + },
> + { /* 3 */
> + .exec_handler_fn = message_handler_req_exec_votequorum_qdevice_reconfigure,
> + .exec_endian_convert_fn = exec_votequorum_qdevice_reconfigure_endian_convert
> + },
> };
>
> /*
> @@ -1144,11 +1182,6 @@ static int votequorum_exec_send_nodeinfo(uint32_t nodeid)
> req_exec_quorum_nodeinfo.votes = node->votes;
> req_exec_quorum_nodeinfo.expected_votes = node->expected_votes;
> req_exec_quorum_nodeinfo.flags = node->flags;
> - if ((nodeid != NODEID_QDEVICE) && (qdevice_is_registered)) {
> - strcpy(req_exec_quorum_nodeinfo.qdevice_name, qdevice->qdevice_name);
> - } else {
> - memset(req_exec_quorum_nodeinfo.qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
> - }
>
> req_exec_quorum_nodeinfo.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_NODEINFO);
> req_exec_quorum_nodeinfo.header.size = sizeof(req_exec_quorum_nodeinfo);
> @@ -1162,6 +1195,50 @@ static int votequorum_exec_send_nodeinfo(uint32_t nodeid)
> return ret;
> }
>
> +static int votequorum_exec_send_qdevice_reconfigure(const char *oldname, const char *newname)
> +{
> + struct req_exec_quorum_qdevice_reconfigure req_exec_quorum_qdevice_reconfigure;
> + struct iovec iov[1];
> + int ret;
> +
> + ENTER();
> +
> + req_exec_quorum_qdevice_reconfigure.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_RECONFIGURE);
> + req_exec_quorum_qdevice_reconfigure.header.size = sizeof(req_exec_quorum_qdevice_reconfigure);
> + strcpy(req_exec_quorum_qdevice_reconfigure.oldname, oldname);
> + strcpy(req_exec_quorum_qdevice_reconfigure.newname, newname);
> +
> + iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reconfigure;
> + iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reconfigure);
> +
> + ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
> +
> + LEAVE();
> + return ret;
> +}
> +
> +static int votequorum_exec_send_qdevice_reg(uint32_t operation, const char *qdevice_name_req)
> +{
> + struct req_exec_quorum_qdevice_reg req_exec_quorum_qdevice_reg;
> + struct iovec iov[1];
> + int ret;
> +
> + ENTER();
> +
> + req_exec_quorum_qdevice_reg.header.id = SERVICE_ID_MAKE(VOTEQUORUM_SERVICE, MESSAGE_REQ_EXEC_VOTEQUORUM_QDEVICE_REG);
> + req_exec_quorum_qdevice_reg.header.size = sizeof(req_exec_quorum_qdevice_reg);
> + req_exec_quorum_qdevice_reg.operation = operation;
> + strcpy(req_exec_quorum_qdevice_reg.qdevice_name, qdevice_name_req);
> +
> + iov[0].iov_base = (void *)&req_exec_quorum_qdevice_reg;
> + iov[0].iov_len = sizeof(req_exec_quorum_qdevice_reg);
> +
> + ret = corosync_api->totem_mcast (iov, 1, TOTEM_AGREED);
> +
> + LEAVE();
> + return ret;
> +}
> +
> static int votequorum_exec_send_quorum_notification(void *conn, uint64_t context)
> {
> struct res_lib_votequorum_notification *res_lib_votequorum_notification;
> @@ -1251,6 +1328,136 @@ static void votequorum_exec_send_expectedvotes_notification(void)
> LEAVE();
> }
>
> +static void exec_votequorum_qdevice_reconfigure_endian_convert (void *message)
> +{
> + ENTER();
> +
> + LEAVE();
> +}
> +
> +static void message_handler_req_exec_votequorum_qdevice_reconfigure (
> + const void *message,
> + unsigned int nodeid)
> +{
> + const struct req_exec_quorum_qdevice_reconfigure *req_exec_quorum_qdevice_reconfigure = message;
> +
> + ENTER();
> +
> + log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice name change req from node %u [from: %s to: %s]",
> + nodeid,
> + req_exec_quorum_qdevice_reconfigure->oldname,
> + req_exec_quorum_qdevice_reconfigure->newname);
> +
> + if (!strcmp(req_exec_quorum_qdevice_reconfigure->oldname, qdevice_name)) {
> + log_printf(LOGSYS_LEVEL_DEBUG, "Allowing qdevice rename");
> + memset(qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
> + strcpy(qdevice_name, req_exec_quorum_qdevice_reconfigure->newname);
> + /*
> + * TODO: notify qdevices about name change?
> + * this is not relevant for now and can wait later on since
> + * qdevices are local only and libvotequorum is not final
> + */
> + }
> +
> + LEAVE();
> +}
> +
> +static void exec_votequorum_qdevice_reg_endian_convert (void *message)
> +{
> + struct req_exec_quorum_qdevice_reg *req_exec_quorum_qdevice_reg = message;
> +
> + ENTER();
> +
> + req_exec_quorum_qdevice_reg->operation = swab32(req_exec_quorum_qdevice_reg->operation);
> +
> + LEAVE();
> +}
> +
> +static void message_handler_req_exec_votequorum_qdevice_reg (
> + const void *message,
> + unsigned int nodeid)
> +{
> + const struct req_exec_quorum_qdevice_reg *req_exec_quorum_qdevice_reg = message;
> + struct res_lib_votequorum_status res_lib_votequorum_status;
> + int wipe_qdevice_name = 1;
> + struct cluster_node *node = NULL;
> + struct list_head *tmp;
> + cs_error_t error = CS_OK;
> +
> + ENTER();
> +
> + log_printf(LOGSYS_LEVEL_DEBUG, "Received qdevice op %u req from node %u [%s]",
> + req_exec_quorum_qdevice_reg->operation,
> + nodeid, req_exec_quorum_qdevice_reg->qdevice_name);
> +
> + switch(req_exec_quorum_qdevice_reg->operation)
> + {
> + case VOTEQUORUM_QDEVICE_OPERATION_REGISTER:
> + if (nodeid != us->node_id) {
> + if (!strlen(qdevice_name)) {
> + log_printf(LOGSYS_LEVEL_DEBUG, "Remote qdevice name recorded");
> + strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name);
> + }
> + LEAVE();
> + return;
> + }
> +
> + /*
> + * this should NEVER happen
> + */
> + if (!qdevice_reg_conn) {
> + log_printf(LOGSYS_LEVEL_WARNING, "Unable to determine origin of the qdevice register call!");
> + LEAVE();
> + return;
> + }
> +
> + /*
> + * registering our own device in this case
> + */
> + if (!strlen(qdevice_name)) {
> + strcpy(qdevice_name, req_exec_quorum_qdevice_reg->qdevice_name);
> + }
> +
> + /*
> + * check if it is our device or something else
> + */
> + if ((!strncmp(req_exec_quorum_qdevice_reg->qdevice_name,
> + qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN))) {
> + qdevice_is_registered = 1;
> + us->flags |= NODE_FLAGS_QDEVICE;
> + votequorum_exec_send_nodeinfo(NODEID_QDEVICE);
> + votequorum_exec_send_nodeinfo(us->node_id);
> + } else {
> + log_printf(LOGSYS_LEVEL_WARNING,
> + "A new qdevice with different name (new: %s old: %s) is trying to register!",
> + req_exec_quorum_qdevice_reg->qdevice_name, qdevice_name);
> + error = CS_ERR_EXIST;
> + }
> +
> + res_lib_votequorum_status.header.size = sizeof(res_lib_votequorum_status);
> + res_lib_votequorum_status.header.id = MESSAGE_RES_VOTEQUORUM_STATUS;
> + res_lib_votequorum_status.header.error = error;
> + corosync_api->ipc_response_send(qdevice_reg_conn, &res_lib_votequorum_status, sizeof(res_lib_votequorum_status));
> + qdevice_reg_conn = NULL;
> + break;
> + case VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER:
> + list_iterate(tmp, &cluster_members_list) {
> + node = list_entry(tmp, struct cluster_node, list);
> + if ((node->state == NODESTATE_MEMBER) &&
> + (node->flags & NODE_FLAGS_QDEVICE)) {
> + wipe_qdevice_name = 0;
> + }
> + }
> +
> + if (wipe_qdevice_name) {
> + memset(qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
> + }
> +
> + break;
> + }
> + LEAVE();
> +}
> +
> static void exec_votequorum_nodeinfo_endian_convert (void *message)
> {
> struct req_exec_quorum_nodeinfo *nodeinfo = message;
> @@ -1317,19 +1524,6 @@ static void message_handler_req_exec_votequorum_nodeinfo (
>
> if (nodeid != NODEID_QDEVICE) {
> node->votes = req_exec_quorum_nodeinfo->votes;
> -
> - memset(node->qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
> - if (node->flags & NODE_FLAGS_QDEVICE) {
> - int state = 0;
> -
> - if (node->flags & NODE_FLAGS_QDEVICE_STATE) {
> - state = 1;
> - }
> -
> - strcpy(node->qdevice_name, req_exec_quorum_nodeinfo->qdevice_name);
> - log_printf(LOGSYS_LEVEL_DEBUG, "Got qdevice data from node[%u]: state %u name %s",
> - node->node_id, state, node->qdevice_name);
> - }
> } else {
> if ((!cluster_is_quorate) &&
> (req_exec_quorum_nodeinfo->flags & NODE_FLAGS_QUORATE)) {
> @@ -1500,7 +1694,7 @@ static char *votequorum_exec_init_fn (struct corosync_api_v1 *api)
> }
> qdevice->state = NODESTATE_DEAD;
> qdevice->votes = 0;
> - memset(qdevice->qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
> + memset(qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
>
> votequorum_readconfig_dynamic();
> recalculate_quorum(0, 0);
> @@ -1580,6 +1774,10 @@ static void votequorum_confchg_fn (
> quorum_members_entries = member_list_entries;
> votequorum_exec_send_nodeinfo(us->node_id);
> votequorum_exec_send_nodeinfo(NODEID_QDEVICE);
> + if (strlen(qdevice_name)) {
> + votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER,
> + qdevice_name);
> + }
> }
>
> memcpy(&quorum_ringid, ring_id, sizeof(*ring_id));
> @@ -1677,7 +1875,7 @@ static void qdevice_timer_fn(void *arg)
>
> qdevice->state = NODESTATE_DEAD;
> us->flags &= ~NODE_FLAGS_QDEVICE_STATE;
> - log_printf(LOGSYS_LEVEL_INFO, "lost contact with quorum device %s", qdevice->qdevice_name);
> + log_printf(LOGSYS_LEVEL_INFO, "lost contact with quorum device %s", qdevice_name);
> votequorum_exec_send_nodeinfo(us->node_id);
>
> qdevice_timer_set = 0;
> @@ -1932,21 +2130,33 @@ static void message_handler_req_lib_votequorum_qdevice_register (void *conn,
>
> if (qdevice_is_registered) {
> if ((!strncmp(req_lib_votequorum_qdevice_register->name,
> - qdevice->qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN))) {
> + qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN))) {
> goto out;
> } else {
> log_printf(LOGSYS_LEVEL_WARNING,
> "A new qdevice with different name (new: %s old: %s) is trying to re-register!",
> - req_lib_votequorum_qdevice_register->name, qdevice->qdevice_name);
> + req_lib_votequorum_qdevice_register->name, qdevice_name);
> error = CS_ERR_EXIST;
> goto out;
> }
> } else {
> - qdevice_is_registered = 1;
> - strcpy(qdevice->qdevice_name, req_lib_votequorum_qdevice_register->name);
> - us->flags |= NODE_FLAGS_QDEVICE;
> - votequorum_exec_send_nodeinfo(NODEID_QDEVICE);
> - votequorum_exec_send_nodeinfo(us->node_id);
> + if (qdevice_reg_conn != NULL) {
> + log_printf(LOGSYS_LEVEL_WARNING,
> + "Registration request already in progress");
> + error = CS_ERR_TRY_AGAIN;
> + goto out;
> + }
> + qdevice_reg_conn = conn;
> + if (votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_REGISTER,
> + req_lib_votequorum_qdevice_register->name) != 0) {
> + log_printf(LOGSYS_LEVEL_WARNING,
> + "Unable to send qdevice registration request to cluster");
> + error = CS_ERR_TRY_AGAIN;
> + qdevice_reg_conn = NULL;
> + } else {
> + LEAVE();
> + return;
> + }
> }
>
> out:
> @@ -1969,7 +2179,7 @@ static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn,
> ENTER();
>
> if (qdevice_is_registered) {
> - if (strncmp(req_lib_votequorum_qdevice_unregister->name, qdevice->qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
> + if (strncmp(req_lib_votequorum_qdevice_unregister->name, qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
> error = CS_ERR_INVALID_PARAM;
> goto out;
> }
> @@ -1981,8 +2191,9 @@ static void message_handler_req_lib_votequorum_qdevice_unregister (void *conn,
> us->flags &= ~NODE_FLAGS_QDEVICE;
> us->flags &= ~NODE_FLAGS_QDEVICE_STATE;
> qdevice->state = NODESTATE_DEAD;
> - memset(qdevice->qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
> votequorum_exec_send_nodeinfo(us->node_id);
> + votequorum_exec_send_qdevice_reg(VOTEQUORUM_QDEVICE_OPERATION_UNREGISTER,
> + req_lib_votequorum_qdevice_unregister->name);
> } else {
> error = CS_ERR_NOT_EXIST;
> }
> @@ -2006,13 +2217,12 @@ static void message_handler_req_lib_votequorum_qdevice_update (void *conn,
> ENTER();
>
> if (qdevice_is_registered) {
> - if (strncmp(req_lib_votequorum_qdevice_update->oldname, qdevice->qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
> + if (strncmp(req_lib_votequorum_qdevice_update->oldname, qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
> error = CS_ERR_INVALID_PARAM;
> goto out;
> }
> - memset(qdevice->qdevice_name, 0, VOTEQUORUM_MAX_QDEVICE_NAME_LEN);
> - strcpy(qdevice->qdevice_name, req_lib_votequorum_qdevice_update->newname);
> - votequorum_exec_send_nodeinfo(us->node_id);
> + votequorum_exec_send_qdevice_reconfigure(req_lib_votequorum_qdevice_update->oldname,
> + req_lib_votequorum_qdevice_update->newname);
> } else {
> error = CS_ERR_NOT_EXIST;
> }
> @@ -2041,7 +2251,7 @@ static void message_handler_req_lib_votequorum_qdevice_poll (void *conn,
> }
>
> if (qdevice_is_registered) {
> - if (strncmp(req_lib_votequorum_qdevice_poll->name, qdevice->qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
> + if (strncmp(req_lib_votequorum_qdevice_poll->name, qdevice_name, VOTEQUORUM_MAX_QDEVICE_NAME_LEN)) {
> error = CS_ERR_INVALID_PARAM;
> goto out;
> }
> @@ -2108,7 +2318,7 @@ static void message_handler_req_lib_votequorum_qdevice_getinfo (void *conn,
> } else {
> res_lib_votequorum_qdevice_getinfo.state = 0;
> }
> - strcpy(res_lib_votequorum_qdevice_getinfo.name, node->qdevice_name);
> + strcpy(res_lib_votequorum_qdevice_getinfo.name, qdevice_name);
> } else {
> error = CS_ERR_NOT_EXIST;
> }
> @@ -2121,7 +2331,7 @@ static void message_handler_req_lib_votequorum_qdevice_getinfo (void *conn,
> } else {
> res_lib_votequorum_qdevice_getinfo.state = 0;
> }
> - strcpy(res_lib_votequorum_qdevice_getinfo.name, qdevice->qdevice_name);
> + strcpy(res_lib_votequorum_qdevice_getinfo.name, qdevice_name);
> } else {
> error = CS_ERR_NOT_EXIST;
> }
_______________________________________________
discuss mailing list
discuss@xxxxxxxxxxxx
http://lists.corosync.org/mailman/listinfo/discuss
