Dave find attached some patches to remove the requirement to use both the clm and evs libraries for cluster/dlm and cluster/group openais integration. Instead a function is added to the evs library to determine the local node identifier. Give it a spin and let me know what you think... I also fixed up a few makefile errors where i couldn't compile the dlm and group from CVS. regards -steve
diff -uNr openais.orig/exec/evs.c openais/exec/evs.c --- openais.orig/exec/evs.c 2005-04-28 00:18:38.000000000 -0700 +++ openais/exec/evs.c 2005-04-28 00:00:27.000000000 -0700 @@ -86,6 +86,7 @@ static int message_handler_req_evs_leave (struct conn_info *conn_info, void *message); static int message_handler_req_evs_mcast_joined (struct conn_info *conn_info, void *message); static int message_handler_req_evs_mcast_groups (struct conn_info *conn_info, void *message); +static int message_handler_req_evs_membership_get (struct conn_info *conn_info, void *message); static int evs_exit_fn (struct conn_info *conn_info); @@ -114,6 +115,12 @@ .response_size = sizeof (struct res_lib_evs_mcast_groups), .response_id = MESSAGE_RES_EVS_MCAST_GROUPS, .flow_control = FLOW_CONTROL_REQUIRED + }, + { /* 4 */ + .libais_handler_fn = message_handler_req_evs_membership_get, + .response_size = sizeof (struct res_lib_evs_membership_get), + .response_id = MESSAGE_RES_EVS_MEMBERSHIP_GET, + .flow_control = FLOW_CONTROL_NOT_REQUIRED } }; @@ -396,6 +403,27 @@ return (0); } +static int message_handler_req_evs_membership_get (struct conn_info *conn_info, void *message) +{ + struct res_lib_evs_membership_get res_lib_evs_membership_get; + + res_lib_evs_membership_get.header.size = sizeof (struct res_lib_evs_membership_get); + res_lib_evs_membership_get.header.id = MESSAGE_RES_EVS_MEMBERSHIP_GET; + res_lib_evs_membership_get.header.error = EVS_OK; + res_lib_evs_membership_get.local_addr.s_addr = this_ip->sin_addr.s_addr; + memcpy (&res_lib_evs_membership_get.member_list, + &res_evs_confchg_callback.member_list, + sizeof (res_lib_evs_membership_get.member_list)); + + res_lib_evs_membership_get.member_list_entries = + res_evs_confchg_callback.member_list_entries; + + libais_send_response (conn_info, &res_lib_evs_membership_get, + sizeof (struct res_lib_evs_membership_get)); + + return (0); +} + static int message_handler_req_exec_mcast (void *message, struct in_addr source_addr, int endian_conversion_required) { struct req_exec_evs_mcast *req_exec_evs_mcast = (struct req_exec_evs_mcast *)message; diff -uNr openais.orig/include/ais_msg.h openais/include/ais_msg.h --- openais.orig/include/ais_msg.h 2005-04-28 00:18:38.000000000 -0700 +++ openais/include/ais_msg.h 2005-04-27 23:53:35.000000000 -0700 @@ -55,7 +55,8 @@ MESSAGE_REQ_EVS_JOIN = 0, MESSAGE_REQ_EVS_LEAVE = 1, MESSAGE_REQ_EVS_MCAST_JOINED = 2, - MESSAGE_REQ_EVS_MCAST_GROUPS = 3 + MESSAGE_REQ_EVS_MCAST_GROUPS = 3, + MESSAGE_REQ_EVS_MEMBERSHIP_GET = 4 }; enum res_lib_evs_types { @@ -64,7 +65,8 @@ MESSAGE_RES_EVS_JOIN = 2, MESSAGE_RES_EVS_LEAVE = 3, MESSAGE_RES_EVS_MCAST_JOINED = 4, - MESSAGE_RES_EVS_MCAST_GROUPS = 5 + MESSAGE_RES_EVS_MCAST_GROUPS = 5, + MESSAGE_RES_EVS_MEMBERSHIP_GET = 6 }; enum req_amf_types { @@ -170,6 +172,17 @@ /* data goes here */ }; +struct req_lib_evs_membership_get { + struct req_header header; +}; + +struct res_lib_evs_membership_get { + struct res_header header; + struct in_addr local_addr; + struct in_addr member_list[16]; + int member_list_entries; +}; + struct req_lib_resdis_init { int size; int id; diff -uNr openais.orig/include/evs.h openais/include/evs.h --- openais.orig/include/evs.h 2005-04-28 00:18:38.000000000 -0700 +++ openais/include/evs.h 2005-04-27 23:50:46.000000000 -0700 @@ -157,4 +157,13 @@ struct iovec *iovec, int iov_len); +/* + * Get membership information from evs + */ +evs_error_t evs_membership_get ( + evs_handle_t handle, + struct in_addr *local_addr, + struct in_addr *member_list, + int *member_list_entries); + #endif /* OPENAIS_EVS_H_DEFINED */ diff -uNr openais.orig/include/list2.h openais/include/list2.h --- openais.orig/include/list2.h 1969-12-31 17:00:00.000000000 -0700 +++ openais/include/list2.h 2005-04-27 23:39:04.000000000 -0700 @@ -0,0 +1,325 @@ +/* Copied from include/linux/list.h */ + +#ifndef _LINUX_LIST_H +#define _LINUX_LIST_H + +/** + * container_of - cast a member of a structure out to the containing structure + * + * @ptr: the pointer to the member. + * @type: the type of the container struct this is embedded in. + * @member: the name of the member within the struct. + * + */ +#define container_of(ptr, type, member) ({ \ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) );}) + + +/* + * These are non-NULL pointers that will result in page faults + * under normal circumstances, used to verify that nobody uses + * non-initialized list entries. + */ +#define LIST_POISON1 ((void *) 0x00100100) +#define LIST_POISON2 ((void *) 0x00200200) + +/* + * Simple doubly linked list implementation. + * + * Some of the internal functions ("__xxx") are useful when + * manipulating whole lists rather than single entries, as + * sometimes we already know the next/prev entries and we can + * generate better code by using them directly rather than + * using the generic single-entry routines. + */ + +struct list_head { + struct list_head *next, *prev; +}; + +#define LIST_HEAD_INIT(name) { &(name), &(name) } + +#define LIST_HEAD(name) \ + struct list_head name = LIST_HEAD_INIT(name) + +#define INIT_LIST_HEAD(ptr) do { \ + (ptr)->next = (ptr); (ptr)->prev = (ptr); \ +} while (0) + +/* + * Insert a new entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next) +{ + next->prev = new; + new->next = next; + new->prev = prev; + prev->next = new; +} + +/** + * list_add - add a new entry + * @new: new entry to be added + * @head: list head to add it after + * + * Insert a new entry after the specified head. + * This is good for implementing stacks. + */ +static inline void list_add(struct list_head *new, struct list_head *head) +{ + __list_add(new, head, head->next); +} + +/** + * list_add_tail - add a new entry + * @new: new entry to be added + * @head: list head to add it before + * + * Insert a new entry before the specified head. + * This is useful for implementing queues. + */ +static inline void list_add_tail(struct list_head *new, struct list_head *head) +{ + __list_add(new, head->prev, head); +} + +/* + * Delete a list entry by making the prev/next entries + * point to each other. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_del(struct list_head * prev, struct list_head * next) +{ + next->prev = prev; + prev->next = next; +} + +/** + * list_del - deletes entry from list. + * @entry: the element to delete from the list. + * Note: list_empty on entry does not return true after this, the entry is + * in an undefined state. + */ +static inline void list_del(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); + entry->next = LIST_POISON1; + entry->prev = LIST_POISON2; +} + +/** + * list_del_init - deletes entry from list and reinitialize it. + * @entry: the element to delete from the list. + */ +static inline void list_del_init(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); + INIT_LIST_HEAD(entry); +} + +/** + * list_move - delete from one list and add as another's head + * @list: the entry to move + * @head: the head that will precede our entry + */ +static inline void list_move(struct list_head *list, struct list_head *head) +{ + __list_del(list->prev, list->next); + list_add(list, head); +} + +/** + * list_move_tail - delete from one list and add as another's tail + * @list: the entry to move + * @head: the head that will follow our entry + */ +static inline void list_move_tail(struct list_head *list, + struct list_head *head) +{ + __list_del(list->prev, list->next); + list_add_tail(list, head); +} + +/** + * list_empty - tests whether a list is empty + * @head: the list to test. + */ +static inline int list_empty(const struct list_head *head) +{ + return head->next == head; +} + +/** + * list_empty_careful - tests whether a list is + * empty _and_ checks that no other CPU might be + * in the process of still modifying either member + * + * NOTE: using list_empty_careful() without synchronization + * can only be safe if the only activity that can happen + * to the list entry is list_del_init(). Eg. it cannot be used + * if another CPU could re-list_add() it. + * + * @head: the list to test. + */ +static inline int list_empty_careful(const struct list_head *head) +{ + struct list_head *next = head->next; + return (next == head) && (next == head->prev); +} + +static inline void __list_splice(struct list_head *list, + struct list_head *head) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + struct list_head *at = head->next; + + first->prev = head; + head->next = first; + + last->next = at; + at->prev = last; +} + +/** + * list_splice - join two lists + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice(struct list_head *list, struct list_head *head) +{ + if (!list_empty(list)) + __list_splice(list, head); +} + +/** + * list_splice_init - join two lists and reinitialise the emptied list. + * @list: the new list to add. + * @head: the place to add it in the first list. + * + * The list at @list is reinitialised + */ +static inline void list_splice_init(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) { + __list_splice(list, head); + INIT_LIST_HEAD(list); + } +} + +/** + * list_entry - get the struct for this entry + * @ptr: the &struct list_head pointer. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_struct within the struct. + */ +#define list_entry(ptr, type, member) \ + container_of(ptr, type, member) + +/** + * list_for_each - iterate over a list + * @pos: the &struct list_head to use as a loop counter. + * @head: the head for your list. + */ +#define list_for_each(pos, head) \ + for (pos = (head)->next; pos != (head); pos = pos->next) + +/** + * __list_for_each - iterate over a list + * @pos: the &struct list_head to use as a loop counter. + * @head: the head for your list. + * + * This variant differs from list_for_each() in that it's the + * simplest possible list iteration code, no prefetching is done. + * Use this for code that knows the list to be very short (empty + * or 1 entry) most of the time. + */ +#define __list_for_each(pos, head) \ + for (pos = (head)->next; pos != (head); pos = pos->next) + +/** + * list_for_each_prev - iterate over a list backwards + * @pos: the &struct list_head to use as a loop counter. + * @head: the head for your list. + */ +#define list_for_each_prev(pos, head) \ + for (pos = (head)->prev; pos != (head); pos = pos->prev) + +/** + * list_for_each_safe - iterate over a list safe against removal of list entry + * @pos: the &struct list_head to use as a loop counter. + * @n: another &struct list_head to use as temporary storage + * @head: the head for your list. + */ +#define list_for_each_safe(pos, n, head) \ + for (pos = (head)->next, n = pos->next; pos != (head); \ + pos = n, n = pos->next) + +/** + * list_for_each_entry - iterate over list of given type + * @pos: the type * to use as a loop counter. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry(pos, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_reverse - iterate backwards over list of given type. + * @pos: the type * to use as a loop counter. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry_reverse(pos, head, member) \ + for (pos = list_entry((head)->prev, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.prev, typeof(*pos), member)) + +/** + * list_prepare_entry - prepare a pos entry for use as a start point in + * list_for_each_entry_continue + * @pos: the type * to use as a start point + * @head: the head of the list + * @member: the name of the list_struct within the struct. + */ +#define list_prepare_entry(pos, head, member) \ + ((pos) ? : list_entry(head, typeof(*pos), member)) + +/** + * list_for_each_entry_continue - iterate over list of given type + * continuing after existing point + * @pos: the type * to use as a loop counter. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry_continue(pos, head, member) \ + for (pos = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry + * @pos: the type * to use as a loop counter. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry_safe(pos, n, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member), \ + n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + + +#endif diff -uNr openais.orig/lib/evs.c openais/lib/evs.c --- openais.orig/lib/evs.c 2005-04-28 00:18:38.000000000 -0700 +++ openais/lib/evs.c 2005-04-28 00:14:06.000000000 -0700 @@ -1,7 +1,7 @@ /* * vi: set autoindent tabstop=4 shiftwidth=4 : - * Copyright (c) 2004 MontaVista Software, Inc. + * Copyright (c) 2004-2005 MontaVista Software, Inc. * * All rights reserved. * @@ -539,3 +539,60 @@ return (error); } + +evs_error_t evs_membership_get ( + evs_handle_t handle, + struct in_addr *local_addr, + struct in_addr *member_list, + int *member_list_entries) +{ + evs_error_t error; + struct evs_inst *evs_inst; + struct iovec iov; + struct req_lib_evs_membership_get req_lib_evs_membership_get; + struct res_lib_evs_membership_get res_lib_evs_membership_get; + + error = saHandleInstanceGet (&evs_handle_t_db, handle, (void *)&evs_inst); + if (error != SA_OK) { + return (error); + } + + req_lib_evs_membership_get.header.size = sizeof (struct req_lib_evs_membership_get); + req_lib_evs_membership_get.header.id = MESSAGE_REQ_EVS_MEMBERSHIP_GET; + + iov.iov_base = &req_lib_evs_membership_get; + iov.iov_len = sizeof (struct req_lib_evs_membership_get); + + pthread_mutex_lock (&evs_inst->response_mutex); + + error = saSendMsgReceiveReply (evs_inst->response_fd, &iov, 1, + &res_lib_evs_membership_get, sizeof (struct res_lib_evs_membership_get)); + + pthread_mutex_unlock (&evs_inst->response_mutex); + + if (error != SA_OK) { + printf ("error\n"); + goto error_exit; + } + + error = res_lib_evs_membership_get.header.error; + printf ("members is %d\n", res_lib_evs_membership_get.member_list_entries); + + /* + * Copy results to caller + */ + if (local_addr) { + memcpy (local_addr, &res_lib_evs_membership_get.local_addr, sizeof (struct in_addr)); + } + *member_list_entries = *member_list_entries < res_lib_evs_membership_get.member_list_entries ? + *member_list_entries : res_lib_evs_membership_get.member_list_entries; + if (member_list) { + memcpy (member_list, &res_lib_evs_membership_get.member_list, + *member_list_entries * sizeof (struct in_addr)); + } + +error_exit: + saHandleInstancePut (&evs_handle_t_db, handle); + + return (error); +} diff -uNr openais.orig/test/testevs.c openais/test/testevs.c --- openais.orig/test/testevs.c 2005-04-28 00:18:38.000000000 -0700 +++ openais/test/testevs.c 2005-04-28 00:14:27.000000000 -0700 @@ -98,12 +98,23 @@ evs_error_t result; int i = 0; int fd; + struct in_addr member_list[16]; + struct in_addr local_addr; + int member_list_entries = sizeof (member_list) / sizeof (struct in_addr); result = evs_initialize (&handle, &callbacks); if (result != EVS_OK) { printf ("Couldn't initialize EVS service %d\n", result); exit (0); } + + result = evs_membership_get (handle, &local_addr, member_list, &member_list_entries); + printf ("Current membership from evs_membership_get entries %d\n", member_list_entries); + for (i = 0; i < member_list_entries; i++) { + printf ("Member [%d] is %s\n", i, inet_ntoa (member_list[i])); + } + printf ("local processor from evs_membership_get %s\n", inet_ntoa (local_addr)); + printf ("Init result %d\n", result); result = evs_join (handle, groups, 3); printf ("Join result %d\n", result);
? group/list.h ? group/daemon/groupd ? group/daemon/list2.h Index: group/daemon/Makefile.openais =================================================================== RCS file: /cvs/cluster/cluster/group/daemon/Makefile.openais,v retrieving revision 1.1 diff -u -r1.1 Makefile.openais --- group/daemon/Makefile.openais 19 Apr 2005 07:00:42 -0000 1.1 +++ group/daemon/Makefile.openais 28 Apr 2005 07:33:21 -0000 @@ -10,7 +10,7 @@ ############################################################################### ############################################################################### -CFLAGS+= -g -I. -I../../../openais/include/ +CFLAGS+= -g -I../include -I../../../openais/include/ TARGET=groupd @@ -23,8 +23,7 @@ update.o \ done.o \ recover.o \ - ../../../openais/lib/libevs.a \ - ../../../openais/lib/libSaClm.a + ../../../openais/lib/libevs.a $(CC) $(LDFLAGS) -o $@ $^ main.o: main.c gd_internal.h Index: group/daemon/openais.c =================================================================== RCS file: /cvs/cluster/cluster/group/daemon/openais.c,v retrieving revision 1.1 diff -u -r1.1 openais.c --- group/daemon/openais.c 19 Apr 2005 07:00:42 -0000 1.1 +++ group/daemon/openais.c 28 Apr 2005 07:33:21 -0000 @@ -38,8 +38,6 @@ #include "gd_internal.h" #include "evs.h" -#include "ais_types.h" -#include "saClm.h" extern struct list_head gd_nodes; extern int gd_node_count; @@ -231,42 +229,14 @@ return 0; } -/* All this SaClm stuff just to get the local nodeid which isn't - available through the evs api. */ - -void foo(SaInvocationT i, const SaClmClusterNodeT *node, SaAisErrorT error) -{ -} - -void bar (const SaClmClusterNotificationBufferT *b, SaUint32T n, SaAisErrorT e) -{ -} - -SaClmCallbacksT clm_callbacks = { - .saClmClusterNodeGetCallback = foo, - .saClmClusterTrackCallback = bar -}; - int set_our_nodeid(void) { - SaVersionT version = { 'B', 1, 1 }; - SaClmHandleT handle; - SaClmClusterNodeT node; - int rv; - - rv = saClmInitialize(&handle, &clm_callbacks, &version); - if (rv != SA_OK) { - log_print("saClmInitialize error %d %d", rv, errno); - return rv; - } - - rv = saClmClusterNodeGet(handle, SA_CLM_LOCAL_NODE_ID, 0, &node); - - gd_nodeid = (int) node.nodeId; + struct in_addr addr; + int member_list_entries = 0; - saClmFinalize(handle); + evs_membership_get (eh, &addr, NULL, &member_list_entries); - log_in("member our nodeid %d rv %d", gd_nodeid, rv); + gd_nodeid = addr.s_addr; return 0; }
? dlm/daemon/dlm_controld ? dlm/make/defines.mk Index: dlm/daemon/Makefile.openais =================================================================== RCS file: /cvs/cluster/cluster/dlm/daemon/Makefile.openais,v retrieving revision 1.1 diff -u -r1.1 Makefile.openais --- dlm/daemon/Makefile.openais 18 Apr 2005 10:02:26 -0000 1.1 +++ dlm/daemon/Makefile.openais 28 Apr 2005 07:32:35 -0000 @@ -17,7 +17,7 @@ CFLAGS+= -g -I${incdir} -I${top_srcdir}/config -CFLAGS+= -I../../dlm-kernel/src2/ -I../../../openais/include/ +CFLAGS+= -I../../group/daemon -I../../dlm-kernel/src2/ -I../../../openais/include/ TARGET=dlm_controld Index: dlm/daemon/member_openais.c =================================================================== RCS file: /cvs/cluster/cluster/dlm/daemon/member_openais.c,v retrieving revision 1.1 diff -u -r1.1 member_openais.c --- dlm/daemon/member_openais.c 18 Apr 2005 10:02:26 -0000 1.1 +++ dlm/daemon/member_openais.c 28 Apr 2005 07:32:35 -0000 @@ -20,8 +20,6 @@ #include <arpa/inet.h> #include "evs.h" -#include "ais_types.h" -#include "saClm.h" #define MAX_NODES (256) @@ -120,48 +118,16 @@ return 0; } -/* All this SaClm stuff just to get the local nodeid which isn't - available through the evs api. */ - -void foo(SaInvocationT i, const SaClmClusterNodeT *node, SaAisErrorT error) -{ -} - -void bar (const SaClmClusterNotificationBufferT *b, SaUint32T n, SaAisErrorT e) -{ -} - -SaClmCallbacksT clm_callbacks = { - .saClmClusterNodeGetCallback = foo, - .saClmClusterTrackCallback = bar -}; - int set_our_nodeid(void) { - SaVersionT version = { 'B', 1, 1 }; - SaClmHandleT handle; - SaClmClusterNodeT node; - struct in_addr a; - int rv; + struct in_addr addr; + int member_list_entries = 0; - rv = saClmInitialize(&handle, &clm_callbacks, &version); - if (rv != SA_OK) { - log_error("saClmInitialize error %d %d", rv, errno); - return rv; - } + evs_membership_get (eh, &addr, NULL, &member_list_entries); - rv = saClmClusterNodeGet(handle, SA_CLM_LOCAL_NODE_ID, 0, &node); - if (rv != SA_OK) { - log_error("saClmClusterNodeGet error %d %d", rv, errno); - return rv; - } + gd_nodeid = addr.s_addr; - saClmFinalize(handle); - - a.s_addr = node.nodeId; - do_set_local((int) node.nodeId, &a); - - return 0; + return 0; } static void dummy(struct in_addr source_addr, void *msg, int len)
-- Linux-cluster@xxxxxxxxxx http://www.redhat.com/mailman/listinfo/linux-cluster