I'm creating a simple utility to encrypt and decrypt files using a key pair. I'm on Windows and coding against the 1.1.0 version of Openssl. I can load the key pair and encrypt the file fine, but when I try to decrypt EVP_PKEY_decrypt always returns -1. I traced this to the rsa padding check functions and they are returning -1 but I can't figure out why. I've tried changing the padding from RSA_PKCS1_OAEP_PADDING to RSA_PKCS1_PADDING and still have the same problem. Any insight would be appreciated, here are my encrypt and decrypt functions:
#define FILE_BUFFER_LENGTH 1#define ENC_BUFFER_LENGTH 2048
int encryptfile(EVP_PKEY *key, FILE *srcfp, FILE *tgtfp) { EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key, NULL); char *inbuf; unsigned char *outbuf; size_t in_len = 0; size_t out_len = ENC_BUFFER_LENGTH; int x;
inbuf = (char*)malloc(sizeof(char)*FILE_BUFFER_LENGTH+1); outbuf = (char*)malloc(sizeof(char)*ENC_BUFFER_LENGTH+1);
if (ctx == NULL) { fprintf(stderr, "Error while creating encryption context.\n"); return 0; } if (EVP_PKEY_encrypt_init(ctx) <= 0) { fprintf(stderr, "Error while initializing encryption context.\n"); return 0; } if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0) { fprintf(stderr, "Error while setting encryption padding.\n"); return 0; }
while (1) { in_len = fread(inbuf, 1, FILE_BUFFER_LENGTH, srcfp);
if (in_len == 0) {break;}
if (EVP_PKEY_encrypt(ctx, outbuf, &out_len, inbuf, in_len) <= 0) { fprintf(stderr, "Error while encrypting data.\n"); return 0; }
x = fwrite(outbuf, sizeof(char), in_len, tgtfp);
if (x != in_len) { fprintf(stderr, "Error while writing to target file.\n"); return 0; } }
return 1;}
int decryptfile(EVP_PKEY *key, FILE *srcfp, FILE *tgtfp) { ENGINE *e = ENGINE_new(); EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new(key, NULL); unsigned char *inbuf; unsigned char *outbuf; size_t in_len = 0; size_t out_len = ENC_BUFFER_LENGTH; int x;
inbuf = (char*)malloc(sizeof(char)*FILE_BUFFER_LENGTH + 1); outbuf = (char*)malloc(sizeof(char)*ENC_BUFFER_LENGTH + 1);
if (ctx == NULL) { fprintf(stderr, "Error while creating decryption context.\n"); return 0; }
if (EVP_PKEY_decrypt_init(ctx) <= 0) { fprintf(stderr, "Error while initializing decryption context.\n"); return 0; } if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0) { fprintf(stderr, "Error while setting decryption padding.\n"); return 0; }
while (1) { in_len = fread(inbuf, 1, FILE_BUFFER_LENGTH, srcfp);
if (in_len == 0) { break; }
if (EVP_PKEY_decrypt(ctx, outbuf, &out_len, inbuf, in_len) <= 0) { fprintf(stderr, "Error while decrypting data.\n"); return 0; }
x = fwrite(outbuf, sizeof(char), in_len, tgtfp);
if (x != in_len) { fprintf(stderr, "Error while writing decrypted data to target file.\n"); return 0; } }
return 1;}
-Mike M.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mta.openssl.org/pipermail/openssl-users/attachments/20160913/3f46afc4/attachment.html>
