Hi, I am running apache bench (ab) and ported it with OpenSSL 1.0 support on top of mTCP support https://github.com/vincentmli/mtcp/commit/642835f786aa642ea0f20fe8db9b09054639453a#diff-02f7a72f514e6e0543e832d37450c251 but it breaks after I upgraded OpenSSL 1.0 to OpenSSL 1.1, I found there are quite a few projects having same problems including libevent, libevent has the fix in https://github.com/libevent/libevent/commit/3e9e0a0d46e4508e8782ec3787c6d86bab63046d so I borrowed the fix from libevent and applied the code fix below: ``` #include <openssl/bio.h> #include "openssl-compat.h" #define get_last_socket_error() errno #define clear_socket_error() errno=0 /* clone of openssl crypto/bio/bss_sock.c */ #ifdef HAVE_MTCP static int mtcp_sock_write(BIO *h, const char *buf, int num); static int mtcp_sock_read(BIO *h, char *buf, int size); static int mtcp_sock_puts(BIO *h, const char *str); static long mtcp_sock_ctrl(BIO *h, int cmd, long arg1, void *arg2); static int mtcp_sock_new(BIO *h); static int mtcp_sock_free(BIO *data); int BIO_mtcp_sock_should_retry(int s); /* static BIO_METHOD mtcp_methods_sockp = { BIO_TYPE_SOCKET, "socket", mtcp_sock_write, mtcp_sock_read, mtcp_sock_puts, NULL, mtcp_sock_ctrl, mtcp_sock_new, mtcp_sock_free, NULL, }; BIO_METHOD *BIO_mtcp_s_socket(void) { return (&mtcp_methods_sockp); } */ static BIO_METHOD *mtcp_methods_sockp; static BIO_METHOD * BIO_mtcp_s_socket(void) { if (mtcp_methods_sockp == NULL) { mtcp_methods_sockp = BIO_meth_new(BIO_TYPE_SOCKET, "socket"); if (mtcp_methods_sockp == NULL) return NULL; BIO_meth_set_write(mtcp_methods_sockp, mtcp_sock_write); BIO_meth_set_read(mtcp_methods_sockp, mtcp_sock_read); BIO_meth_set_puts(mtcp_methods_sockp, mtcp_sock_puts); BIO_meth_set_ctrl(mtcp_methods_sockp, mtcp_sock_ctrl); BIO_meth_set_create(mtcp_methods_sockp, mtcp_sock_new); BIO_meth_set_destroy(mtcp_methods_sockp, mtcp_sock_free); } return mtcp_methods_sockp; } /* BIO *BIO_mtcp_new_socket(mctx_t mctx, int fd, int close_flag) { BIO *ret; ret = BIO_new(BIO_mtcp_s_socket()); if (ret == NULL) return (NULL); BIO_set_fd(ret, fd, close_flag); ret->ptr = mctx; return (ret); } */ BIO *BIO_mtcp_new_socket(mctx_t mctx, int fd, int close_flag) { BIO *ret; if(!fd) return NULL; ret = BIO_new(BIO_mtcp_s_socket()); if (ret == NULL) return (NULL); BIO_set_init(ret, 1); BIO_set_fd(ret, fd, close_flag); BIO_set_data(ret, mctx); return (ret); } /* static int mtcp_sock_new(BIO *bi) { bi->init = 0; bi->num = 0; bi->ptr = NULL; bi->flags = 0; return (1); } */ static int mtcp_sock_new(BIO *bi) { BIO_set_init(bi, 0); BIO_set_num(bi, 0); BIO_set_data(bi, NULL); BIO_set_flags(bi, 0); return (1); } /* static int mtcp_sock_free(BIO *a) { if (a == NULL) return (0); mctx_t mctx = (mctx_t)a->ptr; mtcp_close(mctx, a->num); return (1); } */ static int mtcp_sock_free(BIO *a) { if (a == NULL) return (0); mctx_t mctx = (mctx_t)BIO_get_data(a); mtcp_close(mctx, BIO_get_num(a)); return (1); } /* static int mtcp_sock_read(BIO *b, char *out, int outl) { int ret = 0; mctx_t mctx = (mctx_t)b->ptr; if (out != NULL) { clear_socket_error(); //ret = readsocket(b->num, out, outl); ret = mtcp_read(mctx, b->num, out, outl); BIO_clear_retry_flags(b); if (ret <= 0) { if (BIO_mtcp_sock_should_retry(ret)) BIO_set_retry_read(b); } } return (ret); } */ static int mtcp_sock_read(BIO *b, char *out, int outl) { int ret = 0; mctx_t mctx = (mctx_t)BIO_get_data(b); if (out != NULL) { clear_socket_error(); //ret = readsocket(b->num, out, outl); ret = mtcp_read(mctx, BIO_get_num(b), out, outl); BIO_clear_retry_flags(b); if (ret <= 0) { if (BIO_mtcp_sock_should_retry(ret)) BIO_set_retry_read(b); } } return (ret); } /* static int mtcp_sock_write(BIO *b, const char *in, int inl) { int ret; mctx_t mctx = (mctx_t)b->ptr; clear_socket_error(); //ret = writesocket(b->num, in, inl); ret = mtcp_write(mctx, b->num, in, inl); BIO_clear_retry_flags(b); if (ret <= 0) { if (BIO_mtcp_sock_should_retry(ret)) BIO_set_retry_write(b); } return (ret); } */ static int mtcp_sock_write(BIO *b, const char *in, int inl) { int ret; mctx_t mctx = (mctx_t)BIO_get_data(b); clear_socket_error(); //ret = writesocket(b->num, in, inl); ret = mtcp_write(mctx, BIO_get_num(b), in, inl); BIO_clear_retry_flags(b); if (ret <= 0) { if (BIO_mtcp_sock_should_retry(ret)) BIO_set_retry_write(b); } return (ret); } /* static long mtcp_sock_ctrl(BIO *b, int cmd, long num, void *ptr) { long ret = 1; int *ip; switch (cmd) { case BIO_C_SET_FD: mtcp_sock_free(b); b->num = *((int *)ptr); b->shutdown = (int)num; b->init = 1; break; case BIO_C_GET_FD: if (b->init) { ip = (int *)ptr; if (ip != NULL) *ip = b->num; ret = b->num; } else ret = -1; break; case BIO_CTRL_GET_CLOSE: ret = b->shutdown; break; case BIO_CTRL_SET_CLOSE: b->shutdown = (int)num; break; case BIO_CTRL_DUP: case BIO_CTRL_FLUSH: ret = 1; break; default: ret = 0; break; } return (ret); } */ static long mtcp_sock_ctrl(BIO *b, int cmd, long num, void *ptr) { long ret = 1; int *ip; switch (cmd) { case BIO_C_SET_FD: mtcp_sock_free(b); BIO_set_num(b, *((int *)ptr)); BIO_set_shutdown(b, (int)num); BIO_set_init(b, 1); break; case BIO_C_GET_FD: if (BIO_get_init(b)) { ip = (int *)ptr; if (ip != NULL) *ip = BIO_get_num(b); ret = BIO_get_num(b); } else ret = -1; break; case BIO_CTRL_GET_CLOSE: ret = BIO_get_shutdown(b); break; case BIO_CTRL_SET_CLOSE: BIO_set_shutdown(b, (int)num); break; case BIO_CTRL_DUP: case BIO_CTRL_FLUSH: ret = 1; break; default: ret = 0; break; } return (ret); } ``` here is the openssl-compat.h ``` #ifndef OPENSSL_COMPAT_H #define OPENSSL_COMPAT_H #include <openssl/bio.h> #if (OPENSSL_VERSION_NUMBER < 0x10100000L) static inline BIO_METHOD *BIO_meth_new(int type, const char *name) { BIO_METHOD *biom = calloc(1, sizeof(BIO_METHOD)); if (biom != NULL) { biom->type = type; biom->name = name; } return biom; } #define BIO_meth_set_write(b, f) (b)->bwrite = (f) #define BIO_meth_set_read(b, f) (b)->bread = (f) #define BIO_meth_set_puts(b, f) (b)->bputs = (f) #define BIO_meth_set_ctrl(b, f) (b)->ctrl = (f) #define BIO_meth_set_create(b, f) (b)->create = (f) #define BIO_meth_set_destroy(b, f) (b)->destroy = (f) #define BIO_set_init(b, val) (b)->init = (val) #define BIO_set_data(b, val) (b)->ptr = (val) #define BIO_set_shutdown(b, val) (b)->shutdown = (val) #define BIO_set_num(b, val) (b)->num = (val) #define BIO_set_flags(b, val) (b)->flags = (val) #define BIO_get_init(b) (b)->init #define BIO_get_data(b) (b)->ptr #define BIO_get_shutdown(b) (b)->shutdown #define BIO_get_num(b) (b)->num #define BIO_get_flags(b) (b)->flags #define TLS_method SSLv23_method #endif /* (OPENSSL_VERSION_NUMBER < 0x10100000L) */ #endif /* OPENSSL_COMPAT_H */ ``` but I got compiling error ``` make[2]: Entering directory '/usr/src/mtcp/apps/apache_benchmark/support' /usr/src/mtcp/apps/apache_benchmark/srclib/apr/libtool --silent --mode=compile gcc -g -O2 -pthread -I/usr/src/mtcp/mtcp/lib/ -I/usr/src/mtcp/mtcp/src/include/ -DMULTI_THREADED -I/usr/src/mtcp/mtcp/lib/ -I/usr/src/mtcp/mtcp/src/include/ -DMULTI_THREADED -DLINUX -D_REENTRANT -D_GNU_SOURCE -I/usr/src/mtcp/apps/apache_benchmark/srclib/pcre -I. -I/usr/src/mtcp/apps/apache_benchmark/os/unix -I/usr/src/mtcp/apps/apache_benchmark/include -I/usr/src/mtcp/apps/apache_benchmark/srclib/apr/include -I/usr/src/mtcp/apps/apache_benchmark/srclib/apr-util/include -I/usr/src/mtcp/apps/apache_benchmark/srclib/apr-util/xml/expat/lib -I/usr/lib/include -I/usr/src/mtcp/apps/apache_benchmark/modules/ssl -prefer-non-pic -static -c ab.c && touch ab.lo ab.c: In function ʽmtcp_sock_newʼ: ab.c:547:5: warning: implicit declaration of function ʽBIO_set_numʼ; did you mean ʽBIO_set_nextʼ? [-Wimplicit-function-declaration] BIO_set_num(bi, 0); ^~~~~~~~~~~ BIO_set_next ab.c: In function ʽmtcp_sock_freeʼ: ab.c:570:22: warning: implicit declaration of function ʽBIO_get_numʼ; did you mean ʽBIO_get_initʼ? [-Wimplicit-function-declaration] mtcp_close(mctx, BIO_get_num(a)); ^~~~~~~~~~~ BIO_get_init ..........CUT...... /usr/src/mtcp/apps/apache_benchmark/support/ab.c:547: undefined reference to `BIO_set_num' .libs/ab.o: In function `mtcp_sock_free': /usr/src/mtcp/apps/apache_benchmark/support/ab.c:570: undefined reference to `BIO_get_num' .libs/ab.o: In function `mtcp_sock_ctrl': /usr/src/mtcp/apps/apache_benchmark/support/ab.c:703: undefined reference to `BIO_get_num' /usr/src/mtcp/apps/apache_benchmark/support/ab.c:704: undefined reference to `BIO_get_num' /usr/src/mtcp/apps/apache_benchmark/support/ab.c:695: undefined reference to `BIO_set_num' .libs/ab.o: In function `mtcp_sock_read': /usr/src/mtcp/apps/apache_benchmark/support/ab.c:603: undefined reference to `BIO_get_num' .libs/ab.o: In function `mtcp_sock_write': /usr/src/mtcp/apps/apache_benchmark/support/ab.c:638: undefined reference to `BIO_get_num' .libs/ab.o: In function `main': /usr/src/mtcp/apps/apache_benchmark/support/ab.c:2916: undefined reference to `SSLv3_client_method' /usr/src/mtcp/apps/apache_benchmark/support/ab.c:2913: undefined reference to `SSLv2_client_method' /usr/src/mtcp/apps/apache_benchmark/support/ab.c:2978: undefined reference to `CRYPTO_malloc_init' collect2: error: ld returned 1 exit status Makefile:38: recipe for target 'ab' failed make[2]: *** [ab] Error 1 ``` Since I defined BIO_set/get_init/data/shutdown/num in openssl-compat.h, why it only shows undefined reference to `BIO_set/set_num', not others? I am confused
