Hello,
I am writing a Client Server program. In order to achieve the Network
Throughput I was playing around with the Application buffer size and TCP
Buffer size (SO_SNDBUF/SO_RCVBUF) to find the best values for a specific
client-server pair. I am transferring a file of size 748MB from Client to
Server which are in LAN.
I had tested this with Basic TCP and OpenSSL enabled Client Server program
and have measured the time it takes to transfer the file.
1) In Basic TCP Client Server program I could see the gain at the
different combinations of those 2 buffer values.
2) In OpenSSL enabled Client Server I couldn?t see such gain and the
values are too high as compared to Basic TCP Client Server.
Here are few results:
####### TCP Client #######
[root at vl-blg-dv10 dipak]# zsh
[root at vl-blg-dv10]~/dipak# ./tcp-client_1
usage: ./tcp-client_1 <IPAddress> <port> <filename> <bufferSize>
<tcpbufsize>
[root at vl-blg-dv10]~/dipak# time ./tcp-client_1 10.129.85.242 32000 /tmp/748M
16384 65536 >/dev/null
./tcp-client_1 10.129.85.242 32000 /tmp/748M 16384 65536 > /dev/null 0.06s
user 1.47s system 4% cpu 33.930 total
[root at vl-blg-dv10]~/dipak# time ./tcp-client_1 10.129.85.242 32000 /tmp/748M
65536 131072 >/dev/null
./tcp-client_1 10.129.85.242 32000 /tmp/748M 65536 131072 > /dev/null 0.02s
user 1.20s system 14% cpu 8.503 total
[root at vl-blg-dv10]~/dipak# time ./tcp-client_1 10.129.85.242 32000 /tmp/748M
131072 524288 >/dev/null
./tcp-client_1 10.129.85.242 32000 /tmp/748M 131072 524288 > /dev/null
0.01s user 1.06s system 20% cpu 5.154 total
[root at vl-blg-dv10]~/dipak#
####### OpenSSL Enabled Client #######
[root at vl-blg-dv10]~/dipak# ./SSL_Client_3
usage: ./SSL_Client_3 <hostname> <portnum> <FileName> <AppBufferSize>
<TCPBufferSize>
[root at vl-blg-dv10]~/dipak# time ./SSL_Client_3 10.129.85.242 32000 /tmp/748M
16384 65536 > /dev/null
./SSL_Client_3 10.129.85.242 32000 /tmp/748M 16384 65536 > /dev/null 68.20s
user 3.68s system 86% cpu 1:23.19 total
[root at vl-blg-dv10]~/dipak# time ./SSL_Client_3 10.129.85.242 32000 /tmp/748M
65536 131072 > /dev/null
./SSL_Client_3 10.129.85.242 32000 /tmp/748M 65536 131072 > /dev/null
59.22s user 2.20s system 86% cpu 1:11.06 total
[root at vl-blg-dv10]~/dipak# time ./SSL_Client_3 10.129.85.242 32000 /tmp/748M
131072 524288 > /dev/null
./SSL_Client_3 10.129.85.242 32000 /tmp/748M 131072 524288 > /dev/null
62.15s user 2.45s system 88% cpu 1:12.92 total
[root at vl-blg-dv10]~/dipak#
####### SCP #######
With SCP it is taking around 22 Seconds.
[root at vl-blg-dv10]~/dipak# time scp /tmp/748M root at 10.129.85.242:/tmp/
root at 10.129.85.242's password:
748M
100% 748MB 34.0MB/s 00:22
scp /tmp/748M root at 10.129.85.242:/tmp/ 13.66s user 6.42s system 69% cpu
28.784 total
[root at vl-blg-dv10]~/dipak#
I noticed that in case of OpenSSL Enabled Client the time spend in the ?User
Space? is high (around 60 seconds). So I guess this is mostly because of the
encryption activity.
Please find the source code below for reference. Has anyone faced/noticed
such issue? Is there anything wrong I am doing with my OpenSSL code or
anything needs to be tweaked?
Any pointers, suggestions are always welcome....
Thanks,
Dipak
//################SSL-Client.c #######################
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <sys/socket.h>
#include <resolv.h>
#include <netdb.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <signal.h>
#include <netinet/tcp.h>
#include <netinet/ip.h>
#define FAIL -1
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
/* set the local certificate from CertFile */
if (SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0)
{
ERR_print_errors_fp(stderr);
abort();
}
/* set the private key from KeyFile (may be the same as CertFile) */
if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* verify private key */
if ( !SSL_CTX_check_private_key(ctx) )
{
fprintf(stderr, "Private key does not match the public
certificate\n");
abort();
}
}
int OpenConnection(const char *hostname, int port, int tbuf_size)
{ int sd, val;
struct hostent *host;
struct sockaddr_in addr;
if ((host = gethostbyname(hostname)) == NULL)
{
perror(hostname);
abort();
}
sd = socket(PF_INET, SOCK_STREAM, 0);
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = *(long*)(host->h_addr);
val = tbuf_size;
printf ("Using TCP buffer size (SO_SNDBUF/SO_RCVBUF) = <%d>\n", val);
setsockopt (sd, SOL_SOCKET, SO_RCVBUF, (char *)&val, sizeof (val));
setsockopt (sd, SOL_SOCKET, SO_SNDBUF, (char *)&val, sizeof (val));
val = 1;
setsockopt (sd, IPPROTO_TCP, TCP_NODELAY, (char *)&val, sizeof(val));
val = IPTOS_THROUGHPUT | IPTOS_LOWDELAY;
setsockopt (sd, IPPROTO_IP, IP_TOS, (char *)&val, sizeof(val));
if (connect(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0)
{
close(sd);
perror(hostname);
abort();
}
return sd;
}
SSL_CTX* InitCTX(void)
{ SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); /* Load cryptos, et.al. */
SSL_load_error_strings(); /* Bring in and register error messages
*/
method = SSLv2_client_method(); /* Create new client-method instance */
ctx = SSL_CTX_new(method); /* Create new context */
if (ctx == NULL)
{
ERR_print_errors_fp(stderr);
abort();
}
LoadCertificates(ctx, "client_cert.pem", "client_key.pem"); /* load
certs */
return ctx;
}
int main(int count, char *argv[])
{
SSL_CTX *ctx;
int server;
SSL *ssl;
char *hostname, *portnum;
char *recvBuff1;
int fd=0;
int rlen;
long sz=0;
struct stat st;
int read_cnt;
int data_cnt;
int tobe_read_cnt;
int ok;
int app_buf_size = 0;
int tcp_buf_size = 0;
if (count != 6)
{
fprintf(stderr, "usage: %s <hostname> <portnum> <FileName>
<AppBufferSize> <TCPBufferSize>\n", argv[0]);
exit(0);
}
SSL_library_init();
OpenSSL_add_all_algorithms();
hostname=argv[1];
portnum=argv[2];
app_buf_size = atoi (argv[4]);
tcp_buf_size = atoi (argv[5]);
if (stat(argv[3], &st) < 0)
{
fprintf (stderr, "stat() failed. Error = %s\n", strerror(errno));
return -1;
}
printf ("Using Application buffer size = <%d>\n", app_buf_size);
recvBuff1 = malloc (app_buf_size);
data_cnt = st.st_size;
ctx = InitCTX();
server = OpenConnection(hostname, atoi(portnum), tcp_buf_size);
ssl = SSL_new(ctx); /* create new SSL connection state */
SSL_set_fd(ssl, server); /* attach the socket descriptor */
//SSL_CTX_set_cipher_list(ctx, "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");
if (SSL_connect(ssl) == FAIL) /* perform the connection */
{
ERR_print_errors_fp(stderr);
close(server); /* close socket */
SSL_CTX_free(ctx); /* release context */
return -1;
}
printf ("Connected to %s:%s\n", hostname, portnum);
printf ("SSL_get_cipher_name returned <%s>\n",
SSL_get_cipher_name(ssl));
fd = open (argv[3], O_RDONLY);
if (fd == 0)
{
fprintf (stderr, "Failed to open the file <%s>\n", argv[3]);
return 0;
}
read_cnt = 0;
ok = 1;
tobe_read_cnt = app_buf_size;
//Send the Application buffer size to be used for read
SSL_write(ssl, &app_buf_size, sizeof(app_buf_size)); /* encrypt & send
message */
while (1)
{
if (data_cnt < read_cnt + tobe_read_cnt)
tobe_read_cnt = data_cnt - read_cnt;
if((rlen = read(fd, recvBuff1, tobe_read_cnt)) > 0)
{
read_cnt += rlen;
}
else
break;
sz = SSL_write (ssl, recvBuff1,rlen ); /* encrypt & send message
*/
if (sz != rlen)
{
fprintf (stderr, "Write data length <%ld> doesn't match input
data length <%d>\n", sz, rlen);
}
if (data_cnt == read_cnt)
{
printf ("Sucessfully sent data of length %d\n", read_cnt);
ok = 0;
break;
}
}
SSL_free(ssl); /* release connection state */
close(server); /* close socket */
SSL_CTX_free(ctx); /* release context */
return 0;
}
//######################## SSL-Server.c #########################
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <resolv.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
#include <sys/stat.h>
#include <fcntl.h>
#include <netinet/tcp.h>
#include <netinet/ip.h>
#include <sys/types.h>
#include <sys/wait.h>
#define FAIL -1
int non_ssl;
int OpenListener(int port, int tbuf_size)
{ int sd, val;
struct sockaddr_in addr;
sd = socket(PF_INET, SOCK_STREAM, 0);
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_port = htons(port);
addr.sin_addr.s_addr = INADDR_ANY;
val = tbuf_size;
printf ("Using TCP buffer size (SO_SNDBUF/SO_RCVBUF) = <%d>\n", val);
setsockopt (sd, SOL_SOCKET, SO_RCVBUF, (char *)&val, sizeof (val));
setsockopt (sd, SOL_SOCKET, SO_SNDBUF, (char *)&val, sizeof (val));
val = 1;
setsockopt (sd, IPPROTO_TCP, TCP_NODELAY, (char *)&val, sizeof(val));
val = IPTOS_THROUGHPUT | IPTOS_LOWDELAY;
setsockopt (sd, IPPROTO_IP, IP_TOS, (char *)&val, sizeof(val));
if (bind(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0)
{
perror("can't bind port");
abort();
}
if (listen(sd, 64) != 0)
{
perror("Can't configure listening port");
abort();
}
return sd;
}
SSL_CTX* InitServerCTX(void)
{ SSL_METHOD *method;
SSL_CTX *ctx;
OpenSSL_add_all_algorithms(); /* load & register all cryptos, etc. */
SSL_load_error_strings(); /* load all error messages */
method = SSLv2_server_method(); /* create new server-method instance */
ctx = SSL_CTX_new(method); /* create new context from method */
if ( ctx == NULL )
{
ERR_print_errors_fp(stderr);
abort();
}
return ctx;
}
void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
/* set the local certificate from CertFile */
if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0
)
{
ERR_print_errors_fp(stderr);
abort();
}
/* set the private key from KeyFile (may be the same as CertFile) */
if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
{
ERR_print_errors_fp(stderr);
abort();
}
/* verify private key */
if ( !SSL_CTX_check_private_key(ctx) )
{
fprintf(stderr, "Private key does not match the public
certificate\n");
abort();
}
}
void Servlet(SSL* ssl) /* Serve the connection request*/
{ char *buf;
int sd, bytes;
int fd=0;
int len;
char filename [32];
int app_buf_size = 0;
if (SSL_accept(ssl) == FAIL) /* do SSL-protocol accept */
{
ERR_print_errors_fp(stderr);
sd = SSL_get_fd(ssl); /* get socket connection */
SSL_free(ssl); /* release SSL state */
close(sd); /* close connection */
return;
}
sprintf (filename, "/tmp/test1");
fd = open (filename, O_WRONLY|O_CREAT, 0666);
if (fd < 0)
{
fprintf (stderr, "Failed to open the file %s, error = %s", filename,
strerror(errno));
return;
}
//Get Application buffer size to be used.
SSL_read(ssl, &app_buf_size, sizeof(app_buf_size));
printf ("Using Application buffer size as <%d>\n", app_buf_size);
buf = malloc (app_buf_size);
while(1)
{
bytes = SSL_read(ssl, buf, app_buf_size); /* get request */
//bytes = read(non_ssl, buf, sizeof(buf)); /* get request */
if (bytes <= 0)
{
ERR_print_errors_fp (stderr);
break;
}
len=write(fd,buf,bytes);
}
sd = SSL_get_fd(ssl); /* get socket connection */
SSL_free(ssl); /* release SSL state */
close(sd); /* close connection */
close(fd);
free(buf);
unlink(filename);
}
int main(int count, char *argv[])
{ SSL_CTX *ctx;
int server;
int portnum;
int pid;
int tcp_buf_size = 0;
if (count != 3)
{
printf("Usage: %s <portnum> <tcpbufsize>\n", argv[0]);
exit(0);
}
SSL_library_init();
portnum = atoi (argv[1]);
tcp_buf_size = atoi (argv[2]);
server = OpenListener(portnum, tcp_buf_size); /* create server socket
*/
ctx = InitServerCTX(); /* initialize SSL */
LoadCertificates(ctx, "server_cert.pem", "server_key.pem"); /* load
certs */
while (1)
{ struct sockaddr_in addr;
socklen_t len = sizeof(addr);
SSL *ssl;
int client = accept(server, (struct sockaddr*)&addr, &len); /*
accept connection as usual */
if (client < 0)
break;
printf("Accepted Connection: %s:%d\n",inet_ntoa(addr.sin_addr),
ntohs(addr.sin_port));
pid = fork();
if (pid == 0)
{
ssl = SSL_new(ctx); /* get new SSL state with
context */
SSL_set_fd(ssl, client); /* set connection socket to SSL
state */
OpenSSL_add_all_ciphers();
non_ssl=client;
Servlet(ssl); /* service connection */
printf("Done.\n\n");
_exit (0);
}
else
{
//Parent wait for next client
continue;
}
}
SSL_CTX_free(ctx); /* release context */
close(server); /* close server socket */
return 0;
}
--
View this message in context: http://openssl.6102.n7.nabble.com/Application-and-TCP-buffers-with-OpenSSL-tp58723.html
Sent from the OpenSSL - User mailing list archive at Nabble.com.
