Application and TCP buffers with OpenSSL

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Hello,

I am writing a Client Server program. In order to achieve the Network
Throughput I was playing around with the Application buffer size and TCP
Buffer size (SO_SNDBUF/SO_RCVBUF) to find the best values for a specific
client-server pair. I am transferring a file of size 748MB from Client to
Server which are in LAN.

I had tested this with Basic TCP and OpenSSL enabled Client Server program
and have measured the time it takes to transfer the file.
1)    In Basic TCP Client Server program I could see the gain at the
different combinations of those 2 buffer values.
2)    In OpenSSL enabled Client Server I couldn?t see such gain and the
values are too high as compared to Basic TCP Client Server.

Here are few results:
####### TCP Client #######
[root at vl-blg-dv10 dipak]# zsh
[root at vl-blg-dv10]~/dipak# ./tcp-client_1
usage: ./tcp-client_1 <IPAddress> <port> <filename> <bufferSize>
<tcpbufsize>
[root at vl-blg-dv10]~/dipak# time ./tcp-client_1 10.129.85.242 32000 /tmp/748M
16384 65536 >/dev/null
./tcp-client_1 10.129.85.242 32000 /tmp/748M 16384 65536 > /dev/null  0.06s
user 1.47s system 4% cpu 33.930 total
[root at vl-blg-dv10]~/dipak# time ./tcp-client_1 10.129.85.242 32000 /tmp/748M
65536 131072 >/dev/null
./tcp-client_1 10.129.85.242 32000 /tmp/748M 65536 131072 > /dev/null  0.02s
user 1.20s system 14% cpu 8.503 total
[root at vl-blg-dv10]~/dipak# time ./tcp-client_1 10.129.85.242 32000 /tmp/748M
131072 524288 >/dev/null
./tcp-client_1 10.129.85.242 32000 /tmp/748M 131072 524288 > /dev/null 
0.01s user 1.06s system 20% cpu 5.154 total
[root at vl-blg-dv10]~/dipak#


####### OpenSSL Enabled Client #######
[root at vl-blg-dv10]~/dipak# ./SSL_Client_3
usage: ./SSL_Client_3 <hostname> <portnum> <FileName> <AppBufferSize>
<TCPBufferSize>
[root at vl-blg-dv10]~/dipak# time ./SSL_Client_3 10.129.85.242 32000 /tmp/748M
16384 65536 > /dev/null
./SSL_Client_3 10.129.85.242 32000 /tmp/748M 16384 65536 > /dev/null  68.20s
user 3.68s system 86% cpu 1:23.19 total
[root at vl-blg-dv10]~/dipak# time ./SSL_Client_3 10.129.85.242 32000 /tmp/748M
65536 131072 > /dev/null
./SSL_Client_3 10.129.85.242 32000 /tmp/748M 65536 131072 > /dev/null 
59.22s user 2.20s system 86% cpu 1:11.06 total
[root at vl-blg-dv10]~/dipak# time ./SSL_Client_3 10.129.85.242 32000 /tmp/748M
131072 524288 > /dev/null
./SSL_Client_3 10.129.85.242 32000 /tmp/748M 131072 524288 > /dev/null 
62.15s user 2.45s system 88% cpu 1:12.92 total
[root at vl-blg-dv10]~/dipak#


####### SCP  #######
With SCP it is taking around 22 Seconds.
[root at vl-blg-dv10]~/dipak# time scp /tmp/748M root at 10.129.85.242:/tmp/
root at 10.129.85.242's password:
748M                                                                                                                    
100%  748MB  34.0MB/s   00:22
scp /tmp/748M root at 10.129.85.242:/tmp/  13.66s user 6.42s system 69% cpu
28.784 total
[root at vl-blg-dv10]~/dipak#

I noticed that in case of OpenSSL Enabled Client the time spend in the ?User
Space? is high (around 60 seconds). So I guess this is mostly because of the
encryption activity.
Please find the source code below for reference. Has anyone faced/noticed
such issue? Is there anything wrong I am doing with my OpenSSL code or
anything needs to be tweaked?

Any pointers, suggestions are always welcome....

Thanks,
Dipak


//################SSL-Client.c #######################
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <sys/socket.h>
#include <resolv.h>
#include <netdb.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <signal.h>
#include <netinet/tcp.h>
#include <netinet/ip.h>

#define FAIL    -1

void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
 /* set the local certificate from CertFile */
    if (SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0)
    {
        ERR_print_errors_fp(stderr);
        abort();
    }
    /* set the private key from KeyFile (may be the same as CertFile) */
    if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
    {
        ERR_print_errors_fp(stderr);
        abort();
    }
    /* verify private key */
    if ( !SSL_CTX_check_private_key(ctx) )
    {
        fprintf(stderr, "Private key does not match the public
certificate\n");
        abort();
    }
}

int OpenConnection(const char *hostname, int port, int tbuf_size)
{   int sd, val;
    struct hostent *host;
    struct sockaddr_in addr;

    if ((host = gethostbyname(hostname)) == NULL)
    {
        perror(hostname);
        abort();
    }
    sd = socket(PF_INET, SOCK_STREAM, 0);
    bzero(&addr, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(port);
    addr.sin_addr.s_addr = *(long*)(host->h_addr);

    val = tbuf_size;
    printf ("Using TCP buffer size (SO_SNDBUF/SO_RCVBUF) = <%d>\n", val);
    setsockopt (sd, SOL_SOCKET, SO_RCVBUF, (char *)&val, sizeof (val));
    setsockopt (sd, SOL_SOCKET, SO_SNDBUF, (char *)&val, sizeof (val));

    val = 1;
    setsockopt (sd, IPPROTO_TCP, TCP_NODELAY, (char *)&val, sizeof(val));
    val = IPTOS_THROUGHPUT | IPTOS_LOWDELAY;
    setsockopt (sd, IPPROTO_IP, IP_TOS, (char *)&val, sizeof(val));

    if (connect(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0)
    {
        close(sd);
        perror(hostname);
        abort();
    }
    return sd;
}

SSL_CTX* InitCTX(void)
{   SSL_METHOD *method;
    SSL_CTX *ctx;

    OpenSSL_add_all_algorithms();    /* Load cryptos, et.al. */
    SSL_load_error_strings();        /* Bring in and register error messages
*/
    method = SSLv2_client_method();  /* Create new client-method instance */
    ctx = SSL_CTX_new(method);       /* Create new context */
    if (ctx == NULL)
    {
        ERR_print_errors_fp(stderr);
        abort();
    }
    LoadCertificates(ctx, "client_cert.pem", "client_key.pem"); /* load
certs */
    return ctx;
}

int main(int count, char *argv[])
{  
    SSL_CTX *ctx;
    int server;
    SSL *ssl;
    char *hostname, *portnum;
    char *recvBuff1;
    int fd=0;
    int rlen;
    long sz=0;
    struct stat st;
    int read_cnt;
    int data_cnt;
    int tobe_read_cnt;
    int ok;
    int app_buf_size = 0;
    int tcp_buf_size = 0;

    if (count != 6)
    {
        fprintf(stderr, "usage: %s <hostname> <portnum> <FileName>
<AppBufferSize> <TCPBufferSize>\n", argv[0]);
        exit(0);
    }

    SSL_library_init();
    OpenSSL_add_all_algorithms();
    hostname=argv[1];
    portnum=argv[2];
    app_buf_size = atoi (argv[4]);
    tcp_buf_size = atoi (argv[5]);

    if (stat(argv[3], &st) < 0)
    {
        fprintf (stderr, "stat() failed. Error = %s\n", strerror(errno));
        return -1;
    }

    printf ("Using Application buffer size = <%d>\n", app_buf_size);
    recvBuff1 = malloc (app_buf_size);
    data_cnt = st.st_size;
    ctx = InitCTX();
    server = OpenConnection(hostname, atoi(portnum), tcp_buf_size);
    ssl = SSL_new(ctx);      /* create new SSL connection state */
    SSL_set_fd(ssl, server);    /* attach the socket descriptor */
    //SSL_CTX_set_cipher_list(ctx, "ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH");

    if (SSL_connect(ssl) == FAIL)   /* perform the connection */
    {
        ERR_print_errors_fp(stderr);
        close(server);         /* close socket */
        SSL_CTX_free(ctx);        /* release context */
        return -1;
    }
    printf ("Connected to %s:%s\n", hostname, portnum);
    printf ("SSL_get_cipher_name returned <%s>\n",
SSL_get_cipher_name(ssl));
    fd = open (argv[3], O_RDONLY);
    if (fd == 0)
    {
        fprintf (stderr, "Failed to open the file <%s>\n", argv[3]);
        return 0;
    }
    read_cnt = 0;
    ok = 1;
    tobe_read_cnt = app_buf_size;

    //Send the Application buffer size to be used for read
    SSL_write(ssl, &app_buf_size, sizeof(app_buf_size));   /* encrypt & send
message */

    while (1)
    {
        if (data_cnt < read_cnt + tobe_read_cnt)
            tobe_read_cnt = data_cnt - read_cnt;
        if((rlen = read(fd, recvBuff1, tobe_read_cnt)) > 0)
        {
           read_cnt += rlen;
        }
        else
           break;

        sz = SSL_write (ssl, recvBuff1,rlen );   /* encrypt & send message
*/
        if (sz != rlen)
        {
           fprintf (stderr, "Write data length <%ld> doesn't match input
data length <%d>\n", sz, rlen);
        }
        if (data_cnt == read_cnt)
        {
           printf ("Sucessfully sent data of length %d\n", read_cnt);
           ok = 0;
           break;
        }
    }
    SSL_free(ssl);        /* release connection state */
    close(server);         /* close socket */
    SSL_CTX_free(ctx);        /* release context */
    return 0;
}



//######################## SSL-Server.c #########################
#include <errno.h>
#include <unistd.h>
#include <malloc.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <resolv.h>
#include "openssl/ssl.h"
#include "openssl/err.h"
#include <sys/stat.h>
#include <fcntl.h>
#include <netinet/tcp.h>
#include <netinet/ip.h>
#include <sys/types.h>
#include <sys/wait.h>

#define FAIL    -1

int non_ssl;

int OpenListener(int port, int tbuf_size)
{   int sd, val;
    struct sockaddr_in addr;

    sd = socket(PF_INET, SOCK_STREAM, 0);

    bzero(&addr, sizeof(addr));
    addr.sin_family = AF_INET;
    addr.sin_port = htons(port);
    addr.sin_addr.s_addr = INADDR_ANY;

    val = tbuf_size;
    printf ("Using TCP buffer size (SO_SNDBUF/SO_RCVBUF) = <%d>\n", val);
    setsockopt (sd, SOL_SOCKET, SO_RCVBUF, (char *)&val, sizeof (val));
    setsockopt (sd, SOL_SOCKET, SO_SNDBUF, (char *)&val, sizeof (val));

    val = 1;
    setsockopt (sd, IPPROTO_TCP, TCP_NODELAY, (char *)&val, sizeof(val));
    val = IPTOS_THROUGHPUT | IPTOS_LOWDELAY;
    setsockopt (sd, IPPROTO_IP, IP_TOS, (char *)&val, sizeof(val));

    if (bind(sd, (struct sockaddr*)&addr, sizeof(addr)) != 0)
    {
        perror("can't bind port");
        abort();
    }
    if (listen(sd, 64) != 0)
    {
        perror("Can't configure listening port");
        abort();
    }
    return sd;
}

SSL_CTX* InitServerCTX(void)
{   SSL_METHOD *method;
    SSL_CTX *ctx;

    OpenSSL_add_all_algorithms();  /* load & register all cryptos, etc. */
    SSL_load_error_strings();   /* load all error messages */
    method = SSLv2_server_method();  /* create new server-method instance */
    ctx = SSL_CTX_new(method);   /* create new context from method */
    if ( ctx == NULL )
    {
        ERR_print_errors_fp(stderr);
        abort();
    }
    return ctx;
}

void LoadCertificates(SSL_CTX* ctx, char* CertFile, char* KeyFile)
{
    /* set the local certificate from CertFile */
    if ( SSL_CTX_use_certificate_file(ctx, CertFile, SSL_FILETYPE_PEM) <= 0
)
    {
        ERR_print_errors_fp(stderr);
        abort();
    }

    /* set the private key from KeyFile (may be the same as CertFile) */
    if ( SSL_CTX_use_PrivateKey_file(ctx, KeyFile, SSL_FILETYPE_PEM) <= 0 )
    {
        ERR_print_errors_fp(stderr);
        abort();
    }

    /* verify private key */
    if ( !SSL_CTX_check_private_key(ctx) )
    {
        fprintf(stderr, "Private key does not match the public
certificate\n");
        abort();
    }

}


void Servlet(SSL* ssl) /* Serve the connection request*/
{   char *buf;
    int sd, bytes;
    int fd=0;
    int len;
    char filename [32];
    int app_buf_size = 0;

    if (SSL_accept(ssl) == FAIL)     /* do SSL-protocol accept */
    {
        ERR_print_errors_fp(stderr);
        sd = SSL_get_fd(ssl);       /* get socket connection */
        SSL_free(ssl);         /* release SSL state */
        close(sd);          /* close connection */
        return;
    }

    sprintf (filename, "/tmp/test1");

    fd = open (filename, O_WRONLY|O_CREAT, 0666);

    if (fd < 0)
    {
        fprintf (stderr, "Failed to open the file %s, error = %s", filename,
strerror(errno));
        return;
    }

    //Get Application buffer size to be used.
    SSL_read(ssl, &app_buf_size, sizeof(app_buf_size));
    printf ("Using Application buffer size as <%d>\n", app_buf_size);
    buf = malloc (app_buf_size);

    while(1)
    {
        bytes = SSL_read(ssl, buf, app_buf_size); /* get request */
        //bytes = read(non_ssl, buf, sizeof(buf)); /* get request */
        if (bytes <= 0)
        {
            ERR_print_errors_fp (stderr);
            break;
        }

        len=write(fd,buf,bytes);

    }

    sd = SSL_get_fd(ssl);       /* get socket connection */
    SSL_free(ssl);         /* release SSL state */
    close(sd);          /* close connection */
    close(fd);
    free(buf);
    unlink(filename);
}

int main(int count, char *argv[])
{   SSL_CTX *ctx;
    int server;
    int portnum;
    int pid;
    int tcp_buf_size = 0;

    if (count != 3)
    {
        printf("Usage: %s <portnum> <tcpbufsize>\n", argv[0]);
        exit(0);
    }
    SSL_library_init();

    portnum = atoi (argv[1]);
    tcp_buf_size = atoi (argv[2]);
    server = OpenListener(portnum, tcp_buf_size);    /* create server socket
*/

    ctx = InitServerCTX();        /* initialize SSL */
    LoadCertificates(ctx, "server_cert.pem", "server_key.pem"); /* load
certs */

    while (1)
    {   struct sockaddr_in addr;
        socklen_t len = sizeof(addr);
        SSL *ssl;

        int client = accept(server, (struct sockaddr*)&addr, &len);  /*
accept connection as usual */
        if (client < 0)
           break;

        printf("Accepted Connection: %s:%d\n",inet_ntoa(addr.sin_addr),
ntohs(addr.sin_port));
        pid = fork();
        if (pid == 0)
        {
            ssl = SSL_new(ctx);              /* get new SSL state with
context */
            SSL_set_fd(ssl, client);      /* set connection socket to SSL
state */
            OpenSSL_add_all_ciphers();
            non_ssl=client;
            Servlet(ssl);         /* service connection */
            printf("Done.\n\n");
            _exit (0);
        }
        else
        {
            //Parent wait for next client
            continue;
        }
    }

    SSL_CTX_free(ctx);         /* release context */
    close(server);          /* close server socket */
    return 0;
}




--
View this message in context: http://openssl.6102.n7.nabble.com/Application-and-TCP-buffers-with-OpenSSL-tp58723.html
Sent from the OpenSSL - User mailing list archive at Nabble.com.


[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

[Index of Archives]     [Linux ARM Kernel]     [Linux ARM]     [Linux Omap]     [Fedora ARM]     [IETF Annouce]     [Security]     [Bugtraq]     [Linux]     [Linux OMAP]     [Linux MIPS]     [ECOS]     [Asterisk Internet PBX]     [Linux API]

  Powered by Linux