Rgmanager is a heavy user of ccsd, and the method of communication
libccs currently uses can become problematic when we run out (or low on)
reserved ports, causing rgmanager to perform *extremely* badly during
startup (sometimes taking 5-10 minutes to start in reported cases!
Yikes!).
So, here's a patch to fix ccs. This should fix "slow rgmanager"
problems, which was primarily caused when using IPv6 for communications
(the default). It'll also keep your number of available reserved ports
higher ;)
The patch is backwards / forwards compatible (older statically-linked
apps will work with new ccsd, and newly built apps linked against new
libccs will work with old ccsd...).
More information here:
https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=162078
-- Lon
Index: daemon/ccsd.c
===================================================================
RCS file: /cvs/cluster/cluster/ccs/daemon/ccsd.c,v
retrieving revision 1.14.2.4
diff -u -p -r1.14.2.4 ccsd.c
--- daemon/ccsd.c 14 Feb 2005 18:36:14 -0000 1.14.2.4
+++ daemon/ccsd.c 29 Jun 2005 22:58:49 -0000
@@ -20,6 +20,7 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/socket.h>
+#include <sys/un.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <libxml/parser.h>
@@ -29,6 +30,7 @@
#include "cnx_mgr.h"
#include "cluster_mgr.h"
#include "globals.h"
+#include "comm_headers.h"
#include "copyright.cf"
@@ -41,11 +43,12 @@ static int check_cluster_conf(void);
static void daemonize(void);
static void print_start_msg(char *msg);
static int join_group(int sfd, int loopback, int port);
+static int setup_local_socket(int backlog);
int main(int argc, char *argv[]){
int i,error=0;
int trueint = 1;
- int sfds[2] = {-1,-1}, afd;
+ int sfds[3] = {-1,-1,-1}, afd;
struct sockaddr_storage addr;
struct sockaddr_in *addr4 = (struct sockaddr_in *)&addr;
struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)&addr;
@@ -162,9 +165,14 @@ int main(int argc, char *argv[]){
listen(sfds[1], 5);
+ /* Set up the unix (local) socket for CCS lib comms */
+ sfds[2] = setup_local_socket(SOMAXCONN);
+
FD_ZERO(&rset);
FD_SET(sfds[0], &rset);
FD_SET(sfds[1], &rset);
+ if (sfds[2] >= 0)
+ FD_SET(sfds[2], &rset);
while(1){
int len = addr_size;
@@ -178,8 +186,8 @@ int main(int argc, char *argv[]){
continue;
}
- for(i=0; i<2; i++){
- if(!FD_ISSET(sfds[i], &tmp_set)){
+ for(i=0; i<3; i++){
+ if(sfds[i] < 0 || !FD_ISSET(sfds[i], &tmp_set)){
continue;
}
if(i == 0){
@@ -204,6 +212,20 @@ int main(int argc, char *argv[]){
log_err("Error while processing request: %s\n", strerror(-error));
}
close(afd);
+ } else if (i == 2) {
+ log_dbg("NORMAL CCS REQUEST.\n");
+ afd = accept(sfds[i], NULL, NULL);
+ if(afd < 0){
+ log_sys_err("Unable to accept connection");
+ continue;
+ }
+
+ log_dbg("Connection requested from local socket\n");
+
+ if((error = process_request(afd))){
+ log_err("Error while processing request: %s\n", strerror(-error));
+ }
+ close(afd);
} else {
log_dbg("BROADCAST REQUEST.\n");
if((error = process_broadcast(sfds[i]))){
@@ -231,6 +253,7 @@ static void print_usage(FILE *stream){
"Options:\n"
" -4 Use IPv4 only.\n"
" -6 Use IPv6 only.\n"
+ " -I Use IP for everything (disables local sockets)\n"
" -h Help.\n"
" -m <addr> Specify multicast address (\"default\" ok).\n"
" -n No Daemon. Run in the foreground.\n"
@@ -296,11 +319,12 @@ static char *parse_cli_args(int argc, ch
memset(buff, 0, buff_size);
- while((c = getopt(argc, argv, "46cdf:hlm:nP:t:sV")) != -1){
+ while((c = getopt(argc, argv, "46Icdf:hlm:nP:t:sV")) != -1){
switch(c){
case '4':
if(IPv6 == 1){
- fprintf(stderr, "Setting protocol to IPv4 conflicts with multicast address.\n");
+ fprintf(stderr,
+ "Setting protocol to IPv4 conflicts with previous protocol choice.\n");
error = -EINVAL;
goto fail;
}
@@ -310,7 +334,8 @@ static char *parse_cli_args(int argc, ch
break;
case '6':
if(IPv6 == 0){
- fprintf(stderr, "Setting protocol to IPv6 conflicts with multicast address.\n");
+ fprintf(stderr,
+ "Setting protocol to IPv6 conflicts with previous protocol choice.\n");
error = -EINVAL;
goto fail;
}
@@ -318,6 +343,13 @@ static char *parse_cli_args(int argc, ch
buff_index += snprintf(buff+buff_index, buff_size-buff_index,
" IP Protocol:: IPv6 only\n");
break;
+ case 'I':
+ if (use_local) {
+ buff_index += snprintf(buff+buff_index, buff_size-buff_index,
+ " Communication:: Local sockets disabled\n");
+ }
+ use_local = 0;
+ break;
case 'c':
fprintf(stderr, "The '-c' option is depricated.\n"
"Try '-h' for help.\n");
@@ -802,3 +834,42 @@ static int join_group(int sfd, int loopb
EXIT("join_group");
return 0;
}
+
+int setup_local_socket(int backlog)
+{
+ int sock = -1;
+ struct sockaddr_un su;
+ mode_t om;
+
+ ENTER("setup_local_socket");
+ if (use_local == 0)
+ goto fail;
+
+ sock = socket(PF_LOCAL, SOCK_STREAM, 0);
+ if (sock < 0)
+ goto fail;
+
+ /* This is ours ;) */
+ unlink(COMM_LOCAL_SOCKET);
+ om = umask(077);
+ su.sun_family = PF_LOCAL;
+ snprintf(su.sun_path, sizeof(su.sun_path), COMM_LOCAL_SOCKET);
+
+ if (bind(sock, &su, sizeof(su)) < 0) {
+ umask(om);
+ goto fail;
+ }
+ umask(om);
+
+ if (listen(sock, backlog) < 0)
+ goto fail;
+
+ log_dbg("Set up local socket on %s\n", su.sun_path);
+ EXIT("setup_local_socket");
+ return sock;
+fail:
+ if (sock >= 0)
+ close(sock);
+ EXIT("setup_local_socket");
+ return -1;
+}
Index: daemon/globals.c
===================================================================
RCS file: /cvs/cluster/cluster/ccs/daemon/globals.c,v
retrieving revision 1.2.2.1
diff -u -p -r1.2.2.1 globals.c
--- daemon/globals.c 4 Jan 2005 21:59:14 -0000 1.2.2.1
+++ daemon/globals.c 29 Jun 2005 22:58:49 -0000
@@ -22,5 +22,9 @@ int cluster_base_port = 50008;
/* -1 = no preference, 0 = IPv4, 1 = IPv6 */
int IPv6=-1;
+
+/* 1 = allow and use UNIX domain sockets for local ccs queries */
+int use_local = 1;
+
char *multicast_address = NULL;
int ttl=1;
Index: daemon/globals.h
===================================================================
RCS file: /cvs/cluster/cluster/ccs/daemon/globals.h,v
retrieving revision 1.3.2.1
diff -u -p -r1.3.2.1 globals.h
--- daemon/globals.h 4 Jan 2005 21:59:14 -0000 1.3.2.1
+++ daemon/globals.h 29 Jun 2005 22:58:49 -0000
@@ -29,6 +29,7 @@ extern int backend_port;
extern int cluster_base_port;
extern int IPv6;
+extern int use_local;
extern char *multicast_address;
extern int ttl;
#endif /* __GLOBALS_H__ */
Index: include/comm_headers.h
===================================================================
RCS file: /cvs/cluster/cluster/ccs/include/comm_headers.h,v
retrieving revision 1.1.2.3
diff -u -p -r1.1.2.3 comm_headers.h
--- include/comm_headers.h 13 Jan 2005 16:32:50 -0000 1.1.2.3
+++ include/comm_headers.h 29 Jun 2005 22:58:49 -0000
@@ -42,4 +42,6 @@ typedef struct comm_header_s {
int comm_payload_size;
} comm_header_t;
+#define COMM_LOCAL_SOCKET "/var/run/cluster/ccsd.sock"
+
#endif /* __COMM_HEADERS_DOT_H__ */
Index: lib/libccs.c
===================================================================
RCS file: /cvs/cluster/cluster/ccs/lib/libccs.c,v
retrieving revision 1.6.2.3
diff -u -p -r1.6.2.3 libccs.c
--- lib/libccs.c 13 Jan 2005 16:32:51 -0000 1.6.2.3
+++ lib/libccs.c 29 Jun 2005 22:58:49 -0000
@@ -14,6 +14,7 @@
#include <unistd.h>
#include <string.h>
#include <sys/socket.h>
+#include <sys/un.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <errno.h>
@@ -25,7 +26,7 @@
#include <stdio.h>
static int fe_port = 50006;
-static int ipv6=-1;
+static int comm_proto=-1;
static int setup_interface_ipv6(int *sp, int port){
int sock = -1;
@@ -128,6 +129,41 @@ static int setup_interface_ipv4(int *sp,
}
+int
+setup_interface_local(int *sp)
+{
+ struct sockaddr_un sun;
+ int sock = -1, error = 0;
+
+ ENTER("setup_interface_local");
+ sun.sun_family = PF_LOCAL;
+ snprintf(sun.sun_path, sizeof(sun.sun_path), COMM_LOCAL_SOCKET);
+
+ sock = socket(PF_LOCAL, SOCK_STREAM, 0);
+ if (sock < 0) {
+ error = errno;
+ goto fail;
+ }
+
+ error = connect(sock, (struct sockaddr *)&sun, sizeof(sun));
+ if (error < 0) {
+ error = errno;
+ goto fail;
+ }
+
+ *sp = sock;
+ EXIT("setup_interface_local");
+ return PF_LOCAL;
+
+fail:
+ if (sock >= 0){
+ close(sock);
+ }
+ EXIT("setup_interface_local");
+ return -error;
+}
+
+
/**
* setup_interface
* @sp: pointer gets filled in with open socket
@@ -141,31 +177,38 @@ static int setup_interface(int *sp){
int error=-1;
int res_port = IPPORT_RESERVED-1;
int timo=1;
- int ipv4 = (ipv6 != 1)? 1: 0;
+ int ipv4 = (comm_proto < 0) ? 1 : (comm_proto == PF_INET);
+ int ipv6 = (comm_proto < 0) ? 1 : (comm_proto == PF_INET6);
+ int local = (comm_proto < 0) ? 1 : (comm_proto == PF_LOCAL);
ENTER("setup_interface");
srandom(getpid());
- for(; res_port >= 512; res_port--){
- if(ipv6 && !(error = setup_interface_ipv6(sp, res_port))){
- error = AF_INET6;
- break;
- } else if(ipv4 && !(error = setup_interface_ipv4(sp, res_port))){
- error = AF_INET;
- break;
+ /* Try to do a local connect first */
+ if (local && !(error = setup_interface_local(sp)))
+ error = PF_LOCAL;
+
+ if (error < 0) {
+ for(; res_port >= 512; res_port--){
+ if (ipv6 && !(error = setup_interface_ipv6(sp, res_port))){
+ error = PF_INET6;
+ break;
+ } else if (ipv4 && !(error = setup_interface_ipv4(sp, res_port))){
+ error = PF_INET;
+ break;
+ }
+ if(error == -ECONNREFUSED){
+ break;
+ }
+
+ /* Connections could have colided, giving ECONNREFUSED, or **
+ ** the port we are trying to bind to may already be in use **
+ ** and since we don't want to collide again, wait a random **
+ ** amount of time......................................... */
+ timo = random();
+ timo /= (RAND_MAX/4);
+ sleep(timo);
}
- if(error == -ECONNREFUSED){
- break;
- }
-
- /* Connections could have colided, giving ECONNREFUSED, or **
- ** the port we are trying to bind to may already be in use **
- ** and since we don't want to collide again, wait a random **
- ** amount of time......................................... */
-
- timo = random();
- timo /= (RAND_MAX/4);
- sleep(timo);
}
EXIT("setup_interface");
return error;
@@ -184,8 +227,8 @@ static int setup_interface(int *sp){
static int do_request(char *buffer){
int error=0;
int sock=-1;
+ char *proto;
comm_header_t *ch = (comm_header_t *)buffer;
- int addr_size=0;
ENTER("do_request");
@@ -194,16 +237,21 @@ static int do_request(char *buffer){
}
/* In the future, we will only try the protocol that worked first */
- if(ipv6 < 0){
- ipv6 = (error == AF_INET6)? 1: 0;
+ if (comm_proto < 0){
+ if (error == AF_INET) {
+ proto = "IPv4";
+ } else if (error == AF_INET6) {
+ proto = "IPv6";
+ } else if (error == PF_LOCAL) {
+ proto = "Local Domain";
+ } else {
+ proto = "Unknown";
+ }
- log_dbg("Protocol version set to %s.\n", (ipv6)?"IPv6":"IPv4");
+ comm_proto = error;
+ log_dbg("Protocol set to %s.\n", proto);
}
- addr_size = (error == AF_INET6)?
- sizeof(struct sockaddr_in6 *):
- sizeof(struct sockaddr_in *);
-
error = write(sock, buffer, sizeof(comm_header_t)+ch->comm_payload_size);
if(error < 0){
log_dbg("Write to socket failed.\n");
Index: man/ccsd.8
===================================================================
RCS file: /cvs/cluster/cluster/ccs/man/ccsd.8,v
retrieving revision 1.4.2.2
diff -u -p -r1.4.2.2 ccsd.8
--- man/ccsd.8 16 Feb 2005 18:47:12 -0000 1.4.2.2
+++ man/ccsd.8 29 Jun 2005 22:58:49 -0000
@@ -22,10 +22,17 @@ applications. It must be run on each no
.SH OPTIONS
.TP
\fB-4\fP
-Use IPv4 for all communication. By default, IPv6 is tried, then IPv4.
+Use IPv4 for inter-node communication. By default, IPv6 is tried, then IPv4.
.TP
\fB-6\fP
-Use IPv6 for all communication. By default, IPv6 is tried, then IPv4.
+Use IPv6 for inter-node communication. By default, IPv6 is tried, then IPv4.
+.TP
+\fB-I\fP
+Force use of IP for local communication (disables use of UNIX domain sockets).
+If set, \fBccsd\fP will use the specified inter-node communication protocol
+(see the \fB-4\fP and \fB-6\fP options). If one is not specified,
+IPv6 is tried, then IPv4. For backward compatibility, IP connections are
+still allowed even when UNIX domain sockets are available.
.TP
\fB-h\fP
Help. Print out the usage syntax.
--
Linux-cluster@xxxxxxxxxx
http://www.redhat.com/mailman/listinfo/linux-cluster
