Re: Why custom RSA engine is not calling default RSA method?

[Shariful Alam <dipto181@xxxxxxxxx>]

Dear Dmitry,

Thank you for your response. 

Here is the stack trace 

=============================================================================================================================

$ strace openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out msg.enc -engine rsa-engine-new
execve("/usr/bin/openssl", ["openssl", "rsautl", "-encrypt", "-inkey", "public.pem", "-pubin", "-in", "msg.txt", "-out", "msg.enc", "-engine", "rsa-engine-new"], [/* 65 vars */]) = 0
brk(NULL)                               = 0x1634000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=103650, ...}) = 0
mmap(NULL, 103650, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3997ce3000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/opt/openssl_test/lib/libssl.so.1.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\327\1\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1713040, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997ce2000
mmap(NULL, 2836448, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3997823000
mprotect(0x7f39978cb000, 2097152, PROT_NONE) = 0
mmap(0x7f3997acb000, 53248, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xa8000) = 0x7f3997acb000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/opt/openssl_test/lib/libcrypto.so.1.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0p\7\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=6286208, ...}) = 0
mmap(NULL, 5508104, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f39972e2000
mprotect(0x7f39975f3000, 2093056, PROT_NONE) = 0
mmap(0x7f39977f2000, 188416, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x310000) = 0x7f39977f2000
mmap(0x7f3997820000, 11272, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3997820000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260`\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=138696, ...}) = 0
mmap(NULL, 2212904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f39970c5000
mprotect(0x7f39970dd000, 2093056, PROT_NONE) = 0
mmap(0x7f39972dc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f39972dc000
mmap(0x7f39972de000, 13352, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f39972de000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\t\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1868984, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997ce1000
mmap(NULL, 3971488, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3996cfb000
mprotect(0x7f3996ebb000, 2097152, PROT_NONE) = 0
mmap(0x7f39970bb000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c0000) = 0x7f39970bb000
mmap(0x7f39970c1000, 14752, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f39970c1000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\r\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0644, st_size=14608, ...}) = 0
mmap(NULL, 2109680, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3996af7000
mprotect(0x7f3996afa000, 2093056, PROT_NONE) = 0
mmap(0x7f3996cf9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f3996cf9000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997ce0000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997cdf000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3997cde000
arch_prctl(ARCH_SET_FS, 0x7f3997cdf700) = 0
mprotect(0x7f39970bb000, 16384, PROT_READ) = 0
mprotect(0x7f3996cf9000, 4096, PROT_READ) = 0
mprotect(0x7f39972dc000, 4096, PROT_READ) = 0
mprotect(0x7f39977f2000, 176128, PROT_READ) = 0
mprotect(0x7f3997acb000, 32768, PROT_READ) = 0
mprotect(0x6ac000, 4096, PROT_READ)     = 0
mprotect(0x7f3997cfd000, 4096, PROT_READ) = 0
munmap(0x7f3997ce3000, 103650)          = 0
set_tid_address(0x7f3997cdf9d0)         = 6006
set_robust_list(0x7f3997cdf9e0, 24)     = 0
rt_sigaction(SIGRTMIN, {0x7f39970cab50, [], SA_RESTORER|SA_SIGINFO, 0x7f39970d6390}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f39970cabe0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f39970d6390}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
brk(NULL)                               = 0x1634000
brk(0x1655000)                          = 0x1655000
futex(0x7f39978213d8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f39978213e4, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f39978213ec, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821368, FUTEX_WAKE_PRIVATE, 2147483647) = 0
rt_sigaction(SIGPIPE, {SIG_IGN, [PIPE], SA_RESTORER|SA_RESTART, 0x7f3996d304c0}, {SIG_DFL, [], 0}, 8) = 0
futex(0x7f3997821530, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821400, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821408, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f399781f880, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f399781f940, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f399781f94c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f39978213f4, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821438, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997821440, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/opt/openssl_test/ssl/openssl.cnf", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=11162, ...}) = 0
read(3, "#\n# OpenSSL example configuratio"..., 4096) = 4096
read(3, "\n# MASK:XXXX a literal mask valu"..., 4096) = 4096
read(3, "ts=CA:FALSE\n\n# Here are some exa"..., 4096) = 2970
brk(0x1676000)                          = 0x1676000
read(3, "", 4096)                       = 0
close(3)                                = 0
futex(0x7f3997821410, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3997ad7698, FUTEX_WAKE_PRIVATE, 2147483647) = 0
sysinfo({uptime=2961, loads=[30016, 30688, 27392], totalram=16689545216, freeram=12341272576, sharedram=140345344, bufferram=240164864, totalswap=1023406080, freeswap=1023406080, procs=922, totalhigh=0, freehigh=0, mem_unit=1}) = 0
futex(0x7f3997ad7778, FUTEX_WAKE_PRIVATE, 2147483647) = 0
futex(0x7f3996cfa0a8, FUTEX_WAKE_PRIVATE, 2147483647) = 0
open("/opt/openssl_test/lib/engines-1.1/rsa-engine-new.so", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p0\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=143576, ...}) = 0
mmap(NULL, 2241392, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f39968d3000
mprotect(0x7f39968ea000, 2093056, PROT_NONE) = 0
mmap(0x7f3996ae9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f3996ae9000
mmap(0x7f3996aeb000, 45936, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f3996aeb000
close(3)                                = 0
mprotect(0x7f3996ae9000, 4096, PROT_READ) = 0
write(2, "engine \"rsa-engine-new\" set.\n", 29engine "rsa-engine-new" set.
) = 29
open("public.pem", O_RDONLY)            = 3
fstat(3, {st_mode=S_IFREG|0664, st_size=451, ...}) = 0
read(3, "-----BEGIN PUBLIC KEY-----\nMIIBI"..., 4096) = 451
brk(0x1674000)                          = 0x1674000
close(3)                                = 0
open("msg.txt", O_RDONLY)               = 3
open("msg.enc", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 4
fstat(3, {st_mode=S_IFREG|0664, st_size=72, ...}) = 0
read(3, "Hi. This is a test message. Hope"..., 4096) = 72
read(3, "", 4096)                       = 0
--- SIGSEGV {si_signo=SIGSEGV, si_code=SEGV_MAPERR, si_addr=0} ---
+++ killed by SIGSEGV (core dumped) +++
Segmentation fault (core dumped)

=============================================================================================================================

I was trying using gdb to debug the error. I get to until RSA_public_encrypt(), then if I step in, I get the segmentation fault,

==================================================================================================

bt
#0  0x0000000000000000 in ?? ()
#1  0x00007ffff77dbfc0 in RSA_public_encrypt (flen=72, from=0x6d8860 "Hi. This is a test message. Hope to see some performace gain with this.\nme", to=0x6d8a70 "x\353;\367\377\177", rsa=0x6d8540, padding=1) at crypto/rsa/rsa_crpt.c:30
#2  0x00000000004479a7 in rsautl_main (argc=0, argv=0x7fffffffde10) at apps/rsautl.c:248
#3  0x00000000004379fa in do_cmd (prog=0x6d5930, argc=11, argv=0x7fffffffde10) at apps/openssl.c:564
#4  0x0000000000436e4d in main (argc=11, argv=0x7fffffffde10) at apps/openssl.c:183

==================================================================================================

To my understanding, instead of pointing to the default encryption function, it is pointing to  0x0000000000000000, thus causing the segmentation fault. 

Thanks,
Shariful

On Tue, Sep 28, 2021 at 1:40 AM Dmitry Belyavsky <beldmit@xxxxxxxxx> wrote:

Dear Shariful,

Could you please also provide a stack trace of your segfault?

On Tue, Sep 28, 2021 at 1:06 AM Shariful Alam <dipto181@xxxxxxxxx> wrote:

Hello,

I have the following simple RSA engine code from e_dasync.c.  Following code compiles and works. Until now, I was under the impression that if I do not supply an alternative method in function bind_dasync(), the engine will use the default method. However, it doesn't seem to be the case. If I comment out line 37 and try to perform an encryption operation with the following command, 

"openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out msg.enc -engine rsa-engine-new" 

I get segmentation errors.

Can anyone please tell me why this is happening?

==============================================================================

1. /* Engine Id and Name */
2. static const char *engine_rsa_id = "rsa-engine-new";
3. static const char *engine_rsa_name = "RSA engine for testing";
4. 
   
5. // data encryption function
6. static int eng_rsa_pub_enc(int flen, const unsigned char *from,
7.                           unsigned char *to, RSA *rsa, int padding) {
8.     printf("Encryption\n");
9.     return 0;
10. }
11. 
    
12. // signature verify
13. static int eng_rsa_pub_dec(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding){
14.     printf("Signature verify:\n");
15.     return 0;
16. }
17. 
    
18. // signature
19. static int eng_rsa_priv_enc(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding){
20.     printf("Signature:\n");
21.     return 0;
22. }
23. 
    
24. // data decryption
25. static int eng_rsa_priv_dec(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding){
26.     printf("Decryption\n");
27.     return 0;
28. }
29. 
    
30. 
    
31. static RSA_METHOD *test_rsa_method = NULL;
32. 
    
33. 
    
34. static int bind_dasync(ENGINE *e){
35.     /* Setup RSA_METHOD */
36.     if ((test_rsa_method = RSA_meth_new("Test RSA method", 0)) == NULL
37.        // || RSA_meth_set_pub_enc(test_rsa_method, eng_rsa_pub_enc) == 0
38.         || RSA_meth_set_pub_dec(test_rsa_method, eng_rsa_pub_dec) == 0
39.         || RSA_meth_set_priv_enc(test_rsa_method, eng_rsa_priv_enc) == 0
40.         || RSA_meth_set_priv_dec(test_rsa_method, eng_rsa_priv_dec) == 0
41.             ) {
42. 
    
43.         return 0;
44.     }
45. 
    
46.     /* Ensure the dasync error handling is set up */
47. 
    
48.     if (!ENGINE_set_id(e, engine_rsa_id)
49.         || !ENGINE_set_name(e, engine_rsa_name)
50.         || !ENGINE_set_RSA(e, test_rsa_method)
51.             ) {
52.         return 0;
53.     }
54.     return 1;
55. }
56. 
    
57. static int bind_helper(ENGINE *e, const char *id){
58.     if (!bind_dasync(e)){
59.         printf("2_Error: Inside Bind helper\n");
60.         return 0;
61.     }
62.     return 1;
63. }
64. 
    
65. IMPLEMENT_DYNAMIC_BIND_FN(bind_helper)
66. IMPLEMENT_DYNAMIC_CHECK_FN()

==============================================================================

Regards,

Shariful Alam

--

SY, Dmitry Belyavsky