Running into trouble -- any attempt to PKCS7_decrypt() S/MIME content that was created with PKCS7_sign()+PKCS7_encrypt() yields an empty result set. I have the distinct impression I'm doing something dumb -- but several days of debugging I'm completely stuck. I've created an MVCE and included it below. The code was built and run on Windows 8.1 Pro, Visual C++ 2008 Express, using OpenSSL 1.1.0-pre6-dev (32-bit build). Interesting point -- If I remove the PKCS7_sign() code, I have no problem encrypting and decrypting the content. I strongly suspect my issue has something todo with S/MIME headers interfering with encryption or decryption. But that theory would suggest there's a bug in OpenSSL's S/MIME handling. I find that hard to swallow -- more likely I'm missing some sort of required flag. One more point -- I understand Sign-then-Encrypt has weaknesses and may not be recommended (http://world.std.com/~dtd/sign_encrypt/sign_encrypt7.html). The reason I'm even trying to debug this is to complete the port of an M2Crypto unittest to OpenSSL 1.1.0, which means the procedure works acceptably under OpenSSL 0.9.8. Any assistance would be greatly appreciated ---------------- Here is the output from running the code >>> Version OpenSSL 1.1.0-pre6-dev xx XXX xxxx (0x10100006) >>> Generate keys and certs Generating a 1024 bit RSA private key ...............++++++ .............................................++++++ writing new private key to 'signer_key.pem' ----- Generating a 1024 bit RSA private key ......................++++++ .........................++++++ writing new private key to 'recipient_key.pem' ----- >>> Read keys and certs >>> Create recipient cert stack (for encryption) >>> Sign >>> Generate signed S/MIME >>> Encrypt >>> Generate encrypted S/MIME >>> Convert S/MIME to PKCS7 >>> Decrypt PKCS7 decrypted text => len -1 '' mismatched response size ---------------- MVCE CODE #include "stdafx.h" #include <string.h> #pragma warning(disable:4996) #include <openssl/applink.c> #include <openssl/bio.h> #include <openssl/err.h> #include <openssl/pem.h> #include <openssl/x509.h> #pragma comment(lib, "libcrypto") EVP_PKEY *load_key(const char fname[]); X509 *load_cert(const char fname[]); int _tmain(int argc, _TCHAR* argv[]) { BIO *mem, *bcont; EVP_PKEY *signer_pkey, *rcpt_pkey; PKCS7 *p7; STACK_OF(X509) *rcpt_stack; X509 *signer_cert, *rcpt_cert; char cleartext[] = "some text to manipulate"; char text[128]; const EVP_CIPHER *cipher = EVP_des_ede3_cbc(); int len, rc; printf(">>> Version %s (0x%lx)\n", OPENSSL_VERSION_TEXT, OPENSSL_VERSION_NUMBER); printf(">>> Generate keys and certs\n"); if ((rc = system("openssl req -new -x509 -nodes " "-subj /C=US/ST=STATE/CN=localhost/emailAddress=signer at example.com " "-newkey rsa:1024 -keyout signer_key.pem -out signer_cert.pem"))) { printf("error %d generating signer cert", rc); return -1; } if ((rc = system("openssl req -new -x509 -nodes " "-subj /C=US/ST=STATE/CN=localhost/emailAddress=recipient at example.com " "-newkey rsa:1024 -keyout recipient_key.pem -out recipient_cert.pem"))) { printf("error %d generating recipient cert", rc); return -1; } printf(">>> Read keys and certs\n"); if (!(signer_pkey = load_key("signer_key.pem")) || !(signer_cert = load_cert("signer_cert.pem")) || !(rcpt_pkey = load_key("recipient_key.pem")) || !(rcpt_cert = load_cert("recipient_cert.pem"))) return -1; printf(">>> Create recipient cert stack (for encryption)\n"); if (!(rcpt_stack = sk_X509_new_null()) || !sk_X509_push(rcpt_stack, rcpt_cert)) { ERR_print_errors_fp(stdout); return -1; } printf(">>> Sign\n"); if (!(mem = BIO_new_mem_buf(cleartext, sizeof(cleartext))) || !(p7 = PKCS7_sign(signer_cert, signer_pkey, NULL, mem, 0))) { ERR_print_errors_fp(stdout); return -1; } BIO_free(mem); printf(">>> Generate signed S/MIME\n"); if (!(mem = BIO_new(BIO_s_mem())) || !SMIME_write_PKCS7(mem, p7, NULL, 0)) { ERR_print_errors_fp(stdout); return -1; } PKCS7_free(p7); printf(">>> Encrypt\n"); BIO_set_mem_eof_return(mem, 0); if (!(p7 = PKCS7_encrypt(rcpt_stack, mem, cipher, 0))) { ERR_print_errors_fp(stdout); return -1; } BIO_free(mem); printf(">>> Generate encrypted S/MIME\n"); if (!(mem = BIO_new(BIO_s_mem())) || !SMIME_write_PKCS7(mem, p7, NULL, 0)) { ERR_print_errors_fp(stdout); return -1; } PKCS7_free(p7); printf(">>> Convert S/MIME to PKCS7\n"); BIO_set_mem_eof_return(mem, 0); if (!(p7 = SMIME_read_PKCS7(mem, &bcont))) { ERR_print_errors_fp(stdout); return -1; } BIO_free(mem); printf(">>> Decrypt PKCS7\n"); if (!(mem = BIO_new(BIO_s_mem())) || !PKCS7_decrypt(p7, rcpt_pkey, rcpt_cert, mem, 0)) { ERR_print_errors_fp(stdout); return -1; } len = BIO_read(mem, text, sizeof(text)); printf("decrypted text => len %d '%s'\n", len, len > 0 ? text : ""); if (len != strlen(cleartext)) printf("mismatched response size\n"); else if (strcmp(cleartext, text) != 0) printf("mismatched content\n"); return 0; } EVP_PKEY *load_key(const char fname[]) { BIO *bio = NULL; EVP_PKEY *key = NULL; if (!(bio = BIO_new_file(fname, "r")) || !(key = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL))) { printf("error reading keyfile: %s\n", fname); ERR_print_errors_fp(stdout); } BIO_free(bio); return key; } X509 *load_cert(const char fname[]) { BIO *bio = NULL; X509 *cert = NULL; if (!(bio = BIO_new_file(fname, "r")) || !(cert = PEM_read_bio_X509(bio, NULL, NULL, NULL))) { printf("error reading certfile: %s\n", fname); ERR_print_errors_fp(stdout); } BIO_free(bio); return cert; } begin 666 smime.p7s M,( &"2J&2(;W#0$'`J" ,( "`0$Q"S )!@4K#@,"&@4`,( &"2J&2(;W#0$' M`0``H((.$3""!#8P@@,>H ,"`0("`0$P#08)*H9(AO<-`0$%!0`P;S$+, D& M`U4$!A,"4T4Q%# 2!@-5! H3"T%D9%1R=7-T($%",28P) 8#500+$QU!9&14 M<G5S="!%>'1E<FYA;"!45% @3F5T=V]R:S$B," &`U4$`Q,9061D5')U<W0@ M17AT97)N86P at 0T$@4F]O=# >%PTP,# U,S Q,#0X,SA:%PTR,# U,S Q,#0X M,SA:,&\Q"S )!@-5! 83`E-%,10P$@8#500*$PM!9&14<G5S="!!0C$F,"0& M`U4$"Q,=061D5')U<W0 at 17AT97)N86P at 5%10($YE='=O<FLQ(C @!@-5! ,3 M&4%D9%1R=7-T($5X=&5R;F%L($-!(%)O;W0P@@$B, T&"2J&2(;W#0$!`04` M`X(!#P`P@@$*`H(!`0"W]QHSYO(`!"TYX$Y;[1^\; _-M?HCML[>FQ$SEZ0I M3'V3G[U*O)/M`QKCC\_E;5!:UI<IE%J L$EZVRZ5_;C*OS<X+1X^D4&M<%;' M\$\_Z#*>=,K(D%3IQE\/>)V:0#P.K&&J7A2/GH>A:E#<UYI.KP6SIG&4G'&S M4& *QQ.=. >&`JCIJ&DF&)"K3+!/(ZLZ3X38W\Z?X6EON]="UVM$Y,>M[FU! M7W):<0 at WLWEEI%F at E#?W`"\-PI)RVM X<ML4J$7$72I]M[36Q.ZLS1-$M\DK MW4,`)?IAN6EJ6",1MZ<SCU9U6?7-*==&MPHK9;;30F\5LKA[^^_I75/5-%HG M`@,!``&C@=PP@=DP'08#51T.!!8$%*V]F'HTM";W^L0F5.\#O> DRU0:, L& M`U4=#P0$`P(!!C /!@-5'1,!`?\$!3 #`0'_,(&9!@-5'2,$@9$P at 8Z %*V] MF'HTM";W^L0F5.\#O> DRU0:H7.D<3!O,0LP"08#500&$P)313$4,!(&`U4$ M"A,+061D5')U<W0 at 04(Q)C D!@-5! L3'4%D9%1R=7-T($5X=&5R;F%L(%14 M4"!.971W;W)K,2(P( 8#500#$QE!9&14<G5S="!%>'1E<FYA;"!#02!2;V]T M@@$!, T&"2J&2(;W#0$!!04``X(!`0"PF^"%)<+6(^(/E@:2G4&8G-F$>8'9 M'EL4!R,V98^PV'>[K$%L1V"#4;#Y,CWG_/8F$\> %J6_6OR'SWAYB2&:XDP' M"H8UO/+>4<32EK?<?D[N</T<.>L,`E$4+8Z]%N#!WT9UYR2M[/1"M(63<!!G MNIT&-4H8TRMZS%%"H7ICT>:[H<4KPC:^$PWFO6-^>7NG"0U JVK=CXK#]O:, M&D(%4=1%]9^G8B%H%2!#/)GG?+TDV*F1%W.(/U8;,3 at 8M'$/FLW(#IZ.+AOA MC)B#RQ\Q\41,Q at 1S279@#\?XO1> :R[IS$P.6IIY#R *+M6>8R8>59*4V((7 M6GO0O,>/3H8$,(($KS""`Y>@`P(!`@(1`. CRQ42 at U.)K6%N>E1G:R$P#08) M*H9(AO<-`0$+!0`P;S$+, D&`U4$!A,"4T4Q%# 2!@-5! H3"T%D9%1R=7-T M($%",28P) 8#500+$QU!9&14<G5S="!%>'1E<FYA;"!45% @3F5T=V]R:S$B M," &`U4$`Q,9061D5')U<W0 at 17AT97)N86P at 0T$@4F]O=# >%PTQ-#$R,C(P M,# P,#!:%PTR,# U,S Q,#0X,SA:,(&;,0LP"08#500&$P)'0C$;,!D&`U4$ M"!,21W)E871E<B!-86YC:&5S=&5R,1 P#@8#500'$P=386QF;W)D,1HP& 8# M500*$Q%#3TU/1$\@0T$@3&EM:71E9#%!,#\&`U4$`Q,X0T]-3T1/(%-(02TR M-38 at 0VQI96YT($%U=&AE;G1I8V%T:6]N(&%N9"!396-U<F4 at 16UA:6P at 0T$P M@@$B, T&"2J&2(;W#0$!`04``X(!#P`P@@$*`H(!`0")L0W:>E,93G!2';Q6 MI at 8FM[A)X);G4:OQ\%H3216CM(P;8+QZ44*G>8RD(M\784Z1U78C"A332@)_ MMAT)@&ZE!#W9NKL6_J&'J2Y#4D,6?*\R4,BF3UKI"-C/DR6<>XCH,&3FI/A6 M@/TJ)!0S%YFL1.5IBZ-&!DO",]3I0)\&L+&LDT"YM0B3.IPJ4Z,0VST at 83Q5 M`X[93G8E`B$I^J-\<79/[N%?@>G[5(#;PWLU4K>$WB(]+# M,7]9O5(WL#-I M+4/K^M:E\9=W9U&,V>XGZ[RE!SAVC*2I./_?C/4#K$F^RO=SF3H/,JN<E3H3 M/0Y&.E=T85"^QD _R^3BGZ(A`@,!``&C@@$7,((!$S ?!@-5'2,$&# 6@!2M MO9AZ-+0F]_K$)E3O`[W@),M4&C =!@-5'0X$%@04DF%K at N&BH*I/[&?QPJ/W MM( `P>PP#@8#51T/`0'_! 0#`@&&,!(&`U4=$P$!_P0(, 8!`?\"`0`P'08# M51TE!!8P% 8(*P8!!04'`P(&""L&`04%!P,$,!$&`U4=( 0*, @P!@8$51T@ M`#!$!@-5'1\$/3 [,#F at -Z UAC-H='1P.B\O8W)L+G5S97)T<G5S="YC;VTO M061D5')U<W1%>'1E<FYA;$-!4F]O="YC<FPP-08(*P8!!04'`0$$*3 G,"4& M""L&`04%!S !AAEH='1P.B\O;V-S<"YU<V5R=')U<W0N8V]M, T&"2J&2(;W M#0$!"P4``X(!`0`;*FZL5<$ZJXC%V.W-5?.J:V$KP D0(YD/Q69J;['UM+5W M7@\"80#??07^$K.D@( `_/L=6VIR`@I!O 6ZP5C5)L+JU4V$^_Z"F,]8&^,B M8YQ2^+L%-JM]6*7>JSMCY=K5<^_LX/M[XJ/_\$(CG,JVC4T^Y$L8`[*H+=38 MNT)+D&F%$-NF-S3H>^ !$*6<RCK'GT^(-&Z*9= :BKNIW,K*-M'T_,)D*36O MUK&G<1'2`T.QCSZ:[)XR4_1VDLJ&- >Y+,KF'$K8F0W!AN*0DOM:0FHC(1#I M9<?UU;M^ZHR%( )BZM$Z!RQ9Q9DS\CB)Y;;I%GH?>13V2A :)OI\BON;,((% M(#""! B@`P(!`@(1`-4+#]T2278FC)\!=Y87SN8P#08)*H9(AO<-`0$+!0`P M at 9LQ"S )!@-5! 83`D=",1LP&08#500($Q)'<F5A=&5R($UA;F-H97-T97(Q M$# .!@-5! <3!U-A;&9O<F0Q&C 8!@-5! H3$4-/34]$3R!#02!,:6UI=&5D M,4$P/P8#500#$SA#3TU/1$\@4TA!+3(U-B!#;&EE;G0 at 075T:&5N=&EC871I M;VX at 86YD(%-E8W5R92!%;6%I;"!#03 >%PTQ-C Q,3,P,# P,#!:%PTQ-S Q M,3(R,S4Y-3E:," Q'C <!@DJADB&]PT!"0$6#VII;4!C87)R;VQL+F-O;3"" M`2(P#08)*H9(AO<-`0$!!0`#@@$/`#""`0H"@@$!`-K\XS'GF('[$TPZLMT= MY]ID(UGI at 9^?K.$F3&?)JS.0Q"6)OD@;8S<+1#[2QFG.S045<BKJ-D6O9FQ\ M<*2_A$&HWT6R`S' 7$<4M7HIO_"G at U#-`1,6W2HZ`,L53(EL?:P_[H%Y/6VB MJU\01/0U<U7T/"K$+CFK\>HV/H^"EPS!W)_L#3<"[3T(BZ3LDTHN"#(\B5A1 M^VO2XN77=+Z\+IU=@1UR!40:,<7&)5,P,O1STRE:UFFYLS65=GVT*:ZY[YK9 M':(_+75)?UCOJQ: M-%=9XH<_VNPXG^;7/:6"2-DDFNH3JMIBVKH$1G/E$ 9 MD8XE<3>#8^@.89*P$#)O+'$"`P$``:."`=<P@@'3,!\&`U4=(P08,!: %))A M:X+AHJ"J3^QG\<*C][2 `,'L,!T&`U4=#@06!!0P:UC0J,N<!7>SB(9<*/G' MV*_ SS .!@-5'0\!`?\$! ,"!: P# 8#51T3`0'_! (P`# =!@-5'24$%C 4 M!@@K!@$%!0<#! 8(*P8!!04'`P(P1 at 8#51T@!#\P/3 [!@PK!@$$`;(Q`0(! M`P4P*S I!@@K!@$%!0<"`18=:'1T<',Z+R]S96-U<F4N8V]M;V1O+FYE="]# M4%,P708#51T?!%8P5#!2H%"@3H9,:'1T<#HO+V-R;"YC;VUO9&]C82YC;VTO M0T]-3T1/4TA!,C4V0VQI96YT075T:&5N=&EC871I;VYA;F1396-U<F5%;6%I M;$-!+F-R;#"!D 8(*P8!!04'`0$$@8,P at 8 P6 8(*P8!!04', *&3&AT=' Z M+R]C<G0N8V]M;V1O8V$N8V]M+T-/34]$3U-(03(U-D-L:65N=$%U=&AE;G1I M8V%T:6]N86YD4V5C=7)E16UA:6Q#02YC<G0P) 8(*P8!!04', &&&&AT=' Z M+R]O8W-P+F-O;6]D;V-A+F-O;3 :!@-5'1$$$S 1 at 0]J:6U 8V%R<F]L;"YC M;VTP#08)*H9(AO<-`0$+!0`#@@$!`&7_YE!"6I-N>DE*'QH34=CM%+[K`1M] M]CL[U/FRY5[^LX>0V\F[3S&JAG>8?S4(\8%YC7"@FZN?&[XNG;*71FB1VC5\ M[C at 1T1/1VFB^.U_DY "31W;:;K"NZ]K)Q3#HO(@&45E,YCJ!NY$AC!C\IGQ: M2/NGP"_K'85*^(.K.&Q*INS)?2E26GN'Y^%BLAID at HA<[DL&']YY*Z 9#&;V MFJ3HYV^Y[HF)FFH-]D/]<5G):'.LJD*"]IJWI4,'-BQ;060E4[7[NKAN!^P\ MBTU&T;&8EQ; '\I'[_^.1-;+K'J.:_]/&2]A0 at L9SC^8NO*8S_4,>"4TRIOH MI'J>$[1$P4TQ@@0C,(($'P(!`3"!L3"!FS$+, D&`U4$!A,"1T(Q&S 9!@-5 M! @3$D=R96%T97(@36%N8VAE<W1E<C$0, X&`U4$!Q,'4V%L9F]R9#$:,!@& M`U4$"A,10T]-3T1/($-!($QI;6ET960Q03 _!@-5! ,3.$-/34]$3R!32$$M M,C4V($-L:65N="!!=71H96YT:6-A=&EO;B!A;F0 at 4V5C=7)E($5M86EL($-! M`A$`U0L/W1))=B:,GP%WEA?.YC )!@4K#@,"&@4`H(("1C 8!@DJADB&]PT! M"0,Q"P8)*H9(AO<-`0<!,!P&"2J&2(;W#0$)!3$/%PTQ-C W,C8Q-#(U,C-: M,",&"2J&2(;W#0$)!#$6!!0<';A(TR 3G49706:]>ZML"&UX$#!;!@DJADB& M]PT!"0\Q3C!,, H&""J&2(;W#0,', X&""J&2(;W#0,"`@(`@# -!@@JADB& M]PT#`@(!0# '!@4K#@,"!S -!@@JADB&]PT#`@(!*# '!@4K#@,"&C"!P at 8) M*P8!! &"-Q $,8&T,(&Q,(&;,0LP"08#500&$P)'0C$;,!D&`U4$"!,21W)E M871E<B!-86YC:&5S=&5R,1 P#@8#500'$P=386QF;W)D,1HP& 8#500*$Q%# M3TU/1$\@0T$@3&EM:71E9#%!,#\&`U4$`Q,X0T]-3T1/(%-(02TR-38 at 0VQI M96YT($%U=&AE;G1I8V%T:6]N(&%N9"!396-U<F4 at 16UA:6P at 0T$"$0#5"P_= M$DEV)HR?`7>6%\[F,('$!@LJADB&]PT!"1 ""S&!M*"!L3"!FS$+, D&`U4$ M!A,"1T(Q&S 9!@-5! @3$D=R96%T97(@36%N8VAE<W1E<C$0, X&`U4$!Q,' M4V%L9F]R9#$:,!@&`U4$"A,10T]-3T1/($-!($QI;6ET960Q03 _!@-5! ,3 M.$-/34]$3R!32$$M,C4V($-L:65N="!!=71H96YT:6-A=&EO;B!A;F0 at 4V5C M=7)E($5M86EL($-!`A$`U0L/W1))=B:,GP%WEA?.YC -!@DJADB&]PT!`0$% M``2"`0`X8UMHRZC;[0?$\65.4:_VF*Q!OS>;K_S5KA<$]28 at A8'M]4H.7L/? M`GQ_N at F,M4"G"CB#1"LUS$1?/$!)@2@#]H,SFZ72ZEH;Y-_)J[SU+1NPB:5N MK at U6X>_F#0<6B VE018PGQZ?#:7L\@H&XI.2/9U[9CZ]R(-$DVN'-_B]7.R< MI$Z:<$'?Q[0`O0/!@=2XEW/USH\\"5&2:'Q2!PUIO^RUSRFV\*MV'AXOSA+L MD at RN,GS$V#E25?E_$CC_H3'J1'0WTQP at OE #PTG#%SUU"" ;UR at FP*]L=AD# J"DEXEF5L^$[[QP8HW3A8=CT;+Q33&2G#"-WT4: 4RIU3,=3^```````` ` end
