On Wed, 1 Jul 2020, Vincent Li wrote: > > Hi, > > I am running apache bench (ab) and ported it with OpenSSL 1.0 support on top of mTCP support > > https://github.com/vincentmli/mtcp/commit/642835f786aa642ea0f20fe8db9b09054639453a#diff-02f7a72f514e6e0543e832d37450c251 > > but it breaks after I upgraded OpenSSL 1.0 to OpenSSL 1.1, I found there > are quite a few projects having same problems including libevent, libevent > has the fix in https://github.com/libevent/libevent/commit/3e9e0a0d46e4508e8782ec3787c6d86bab63046d > > so I borrowed the fix from libevent and applied the code fix below: > > ``` > #include <openssl/bio.h> > #include "openssl-compat.h" > #define get_last_socket_error() errno > #define clear_socket_error() errno=0 > /* clone of openssl crypto/bio/bss_sock.c */ > > #ifdef HAVE_MTCP > > static int mtcp_sock_write(BIO *h, const char *buf, int num); > static int mtcp_sock_read(BIO *h, char *buf, int size); > static int mtcp_sock_puts(BIO *h, const char *str); > static long mtcp_sock_ctrl(BIO *h, int cmd, long arg1, void *arg2); > static int mtcp_sock_new(BIO *h); > static int mtcp_sock_free(BIO *data); > int BIO_mtcp_sock_should_retry(int s); .........SNIP......... > > static long mtcp_sock_ctrl(BIO *b, int cmd, long num, void *ptr) > { > long ret = 1; > int *ip; > > switch (cmd) { > case BIO_C_SET_FD: > mtcp_sock_free(b); > BIO_set_num(b, *((int *)ptr)); > BIO_set_shutdown(b, (int)num); > BIO_set_init(b, 1); > break; > case BIO_C_GET_FD: > if (BIO_get_init(b)) { > ip = (int *)ptr; > if (ip != NULL) > *ip = BIO_get_num(b); > ret = BIO_get_num(b); > } else > ret = -1; > break; > case BIO_CTRL_GET_CLOSE: > ret = BIO_get_shutdown(b); > break; > case BIO_CTRL_SET_CLOSE: > BIO_set_shutdown(b, (int)num); > break; > case BIO_CTRL_DUP: > case BIO_CTRL_FLUSH: > ret = 1; > break; > default: > ret = 0; > break; > } > return (ret); > } I found http://vega.pgw.jp/~kabe/vsd/migrate2openssl-1.1.html which says: "bio->num (file descripter) could be set by BIO_set_fd(), but since this callbacks the routine set by BIO_meth_set_ctrl(biom, BioCtrl), beware of infinite loops. Recommend to not touch bio->num member and leave it alone." so I used BIO_set/get_fd() to work with the bio->num, it compiles ok with some warnings, but it appears I hit the infinite loops and core dump: root@vli-lab:/usr/src/mtcp/apps/apache_benchmark/support# cp .libs/ab . root@vli-lab:/usr/src/mtcp/apps/apache_benchmark/support# gdb --args ab -N 4 -c 4 -n 160 https://10.2.1.63/ GNU gdb (Ubuntu 8.1-0ubuntu3.2) 8.1.0.20180409-git ...........SNIP............ Checking link status..done Port 0 Link Up - speed 10000 Mbps - full-duplex Benchmarking 10.2.1.63 (be patient) [New Thread 0x7fdff19fb700 (LWP 1510)] CPU0 connecting to port 443 [New Thread 0x7fdff11fa700 (LWP 1511)] [New Thread 0x7fdff09f9700 (LWP 1512)] CPU1 connecting to port 443 [New Thread 0x7fdfea6fe700 (LWP 1513)] CPU2 connecting to port 443 [New Thread 0x7fdfd7fff700 (LWP 1514)] CPU3 connecting to port 443 CPU 0: initialization finished. [mtcp_create_context:1359] CPU 0 is now the master thread. [CPU 0] dpdk0 flows: 0, RX: 0(pps) (err: 0), 0.00(Gbps), TX: 0(pps), 0.00(Gbps) [ ALL ] dpdk0 flows: 0, RX: 0(pps) (err: 0), 0.00(Gbps), TX: 0(pps), 0.00(Gbps) Thread 8 "ab" received signal SIGSEGV, Segmentation fault. [Switching to Thread 0x7fdff19fb700 (LWP 1510)] 0x00005555555dd3cd in mtcp_sock_ctrl (b=0x7fdfe4003dd0, cmd=105, num=0, ptr=0x560af680, close_flag=1443559040) at ab.c:692 692 switch (cmd) { (gdb) bt #0 0x00005555555dd3cd in mtcp_sock_ctrl (b=0x7fdfe4003dd0, cmd=105, num=0, ptr=0x560af680, close_flag=1443559040) at ab.c:692 #1 0x00007ffff6474949 in BIO_ctrl () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 #2 0x00005555555dd3fb in mtcp_sock_ctrl (b=0x7fdfe4003dd0, cmd=<optimized out>, num=<optimized out>, ptr=0x560af680, close_flag=1443559040) at ab.c:703 #3 0x00007ffff6474949 in BIO_ctrl () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 #4 0x00005555555dd3fb in mtcp_sock_ctrl (b=0x7fdfe4003dd0, cmd=<optimized out>, num=<optimized out>, ptr=0x560af680, close_flag=1443559040) at ab.c:703 #5 0x00007ffff6474949 in BIO_ctrl () from /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 #6 0x00005555555dd3fb in mtcp_sock_ctrl (b=0x7fdfe4003dd0, cmd=<optimized out>, num=<optimized out>, ptr=0x560af680, close_flag=1443559040) at ab.c:703 ....loop continues.... How do I avoid the loops? here is the changed code: #ifdef USE_SSL #ifndef OPENSSL_NO_SOCK #include <openssl/bio.h> #include "openssl-compat.h" #define get_last_socket_error() errno #define clear_socket_error() errno=0 /* clone of openssl crypto/bio/bss_sock.c */ #ifdef HAVE_MTCP static int mtcp_sock_write(BIO *h, const char *buf, int num, int close_flag); static int mtcp_sock_read(BIO *h, char *buf, int size, int close_flag); static int mtcp_sock_puts(BIO *h, const char *str, int close_flag); static long mtcp_sock_ctrl(BIO *h, int cmd, long arg1, void *arg2, int close_flag); static int mtcp_sock_new(BIO *h); static int mtcp_sock_free(BIO *data, int close_flag); int BIO_mtcp_sock_should_retry(int s); static BIO_METHOD *mtcp_methods_sockp; static BIO_METHOD * BIO_mtcp_s_socket(void) { if (mtcp_methods_sockp == NULL) { mtcp_methods_sockp = BIO_meth_new(BIO_TYPE_SOCKET, "socket"); if (mtcp_methods_sockp == NULL) return NULL; BIO_meth_set_write(mtcp_methods_sockp, mtcp_sock_write); BIO_meth_set_read(mtcp_methods_sockp, mtcp_sock_read); BIO_meth_set_puts(mtcp_methods_sockp, mtcp_sock_puts); BIO_meth_set_ctrl(mtcp_methods_sockp, mtcp_sock_ctrl); BIO_meth_set_create(mtcp_methods_sockp, mtcp_sock_new); BIO_meth_set_destroy(mtcp_methods_sockp, mtcp_sock_free); } return mtcp_methods_sockp; } BIO *BIO_mtcp_new_socket(mctx_t mctx, int fd, int close_flag) { BIO *ret; if(!fd) return NULL; ret = BIO_new(BIO_mtcp_s_socket()); if (ret == NULL) return (NULL); BIO_set_init(ret, 1); BIO_set_fd(ret, fd, close_flag); BIO_set_data(ret, mctx); return (ret); } static int mtcp_sock_new(BIO *bi) { BIO_set_init(bi, 0); // BIO_set_fd(bi, fd, close_flag) BIO_set_data(bi, NULL); BIO_set_flags(bi, 0); return (1); } static int mtcp_sock_free(BIO *a, int close_flag) { if (a == NULL) return (0); mctx_t mctx = (mctx_t)BIO_get_data(a); mtcp_close(mctx, BIO_get_fd(a, close_flag)); return (1); } static int mtcp_sock_read(BIO *b, char *out, int outl, int close_flag) { int ret = 0; mctx_t mctx = (mctx_t)BIO_get_data(b); if (out != NULL) { clear_socket_error(); //ret = readsocket(b->num, out, outl); ret = mtcp_read(mctx, BIO_get_fd(b, close_flag), out, outl); BIO_clear_retry_flags(b); if (ret <= 0) { if (BIO_mtcp_sock_should_retry(ret)) BIO_set_retry_read(b); } } return (ret); } static int mtcp_sock_write(BIO *b, const char *in, int inl, int close_flag) { int ret; mctx_t mctx = (mctx_t)BIO_get_data(b); clear_socket_error(); //ret = writesocket(b->num, in, inl); ret = mtcp_write(mctx, BIO_get_fd(b, close_flag), in, inl); BIO_clear_retry_flags(b); if (ret <= 0) { if (BIO_mtcp_sock_should_retry(ret)) BIO_set_retry_write(b); } return (ret); } static long mtcp_sock_ctrl(BIO *b, int cmd, long num, void *ptr, int close_flag) { long ret = 1; int *ip; switch (cmd) { case BIO_C_SET_FD: mtcp_sock_free(b, close_flag); BIO_set_fd(b, *((int *)ptr), close_flag); BIO_set_shutdown(b, (int)num); BIO_set_init(b, 1); break; case BIO_C_GET_FD: if (BIO_get_init(b)) { ip = (int *)ptr; if (ip != NULL) *ip = BIO_get_fd(b, close_flag); ret = BIO_get_fd(b, close_flag); } else ret = -1; break; case BIO_CTRL_GET_CLOSE: ret = BIO_get_shutdown(b); break; case BIO_CTRL_SET_CLOSE: BIO_set_shutdown(b, (int)num); break; case BIO_CTRL_DUP: case BIO_CTRL_FLUSH: ret = 1; break; default: ret = 0; break; } return (ret); } static int mtcp_sock_puts(BIO *bp, const char *str, int close_flag) { int n, ret; n = strlen(str); ret = mtcp_sock_write(bp, str, n, close_flag); return (ret); }
