Hi,
When we used GlusterFS 2.0.7 with nearly 200 servers and 6
volumes, the client was unable to connect to more than 4 volumes at
the same time. We found that there were no enough ports (200*4=800)
available under port 1024 while client ports must be under 1024. Even
worse, GlusterFS will occupy ports of other programs such as Rsync
port 873. Therefore, we modified code in name.c and used function
setsockopt with SO_REUSEADDR to solve the problem. Below is the patch
by my colleague. It works well now.
--- glusterfs-2.0.7/transport/socket/src/name.c 2009-10-01
21:19:54.000000000 +0800
+++ glusterfs-2.0.7-1117/transport/socket/src/name.c 2009-10-14
19:27:54.000000000 +0800
@@ -22,6 +22,7 @@
#include <errno.h>
#include <netdb.h>
#include <string.h>
+#include <time.h>
#ifdef CLIENT_PORT_CEILING
#undef CLIENT_PORT_CEILING
@@ -49,19 +50,24 @@ af_inet_bind_to_port_lt_ceiling (int fd,
{
int32_t ret = -1;
/* struct sockaddr_in sin = {0, }; */
+ static const uint16_t base = 800; //port bottom number to
avoid occupy some system ports
+ ceiling -= base;
uint16_t port = ceiling - 1;
+ uint16_t off = (time(NULL) + rand()) % ceiling; //random
port between base and ceiling
while (port)
{
+ uint16_t rport = (port+off)%ceiling + base;
+
switch (sockaddr->sa_family)
{
case AF_INET6:
- ((struct sockaddr_in6 *)sockaddr)->sin6_port
= htons (port);
+ ((struct sockaddr_in6 *)sockaddr)->sin6_port
= htons (rport);
break;
case AF_INET_SDP:
case AF_INET:
- ((struct sockaddr_in *)sockaddr)->sin_port =
htons (port);
+ ((struct sockaddr_in *)sockaddr)->sin_port =
htons (rport);
break;
}
@@ -403,6 +409,7 @@ client_bind (transport_t *this,
int sock)
{
int ret = 0;
+ int opt = 1;
*sockaddr_len = sizeof (struct sockaddr_in6);
switch (sockaddr->sa_family)
@@ -412,6 +419,8 @@ client_bind (transport_t *this,
*sockaddr_len = sizeof (struct sockaddr_in);
case AF_INET6:
+ ret = setsockopt (sock, SOL_SOCKET, SO_REUSEADDR,
+ &opt, sizeof (opt));
ret = af_inet_bind_to_port_lt_ceiling (sock, sockaddr,
*sockaddr_len, CLIENT_PORT_CEILING);
if (ret == -1) {
--
Regards,
Jack Wang
Add: PO BOX 181, No.10 XiTuCheng Rd, Beijing 100876, P.R.China
