Problem in RSA encrypt and Decrypt using EVP

[sugu.ece28@xxxxxxxxx (Sugumar)]

Hi,

I have problem in RSA encrypt and decrypt using EVP methods.
My below program is some times working fine and some times it fails to
decrypt the message.
That means when i executing the below code 2 times working fine 3rd time it
fails to decrypt.
Please point out the error.

Code:

#include <iostream>
#include <string>
#include <string.h>
#include <openssl/evp.h>
 #include <openssl/rsa.h>
 #include <openssl/engine.h>

 EVP_PKEY *pkey = EVP_PKEY_new();
 EVP_PKEY_CTX *ctx2;
 unsigned char *out1;
 size_t outlen1;
 EVP_PKEY_CTX *ctx;
 ENGINE *eng;


RSA *rsakey =RSA_new();


bool GenerateRsaKeyPair()
{
	BIGNUM *bnexp = NULL;
	unsigned long exp = RSA_F4;
    bnexp = BN_new();

    if(!BN_set_word(bnexp,exp))
	{
        std::cout <<"Failed to set exponent word in BIO."<<std::endl;
		return false;
    }

    // generate rsa key with length of KEY_LENGTH bits
    RSA_generate_key_ex(rsakey, 1024, bnexp, NULL);

	if(!rsakey)
		return false;

	if(RSA_check_key(rsakey) <= 0)
	{
		std::cout <<"Generated RSA asymmetric key is invalid."<<std::endl;
		return false;
	}
	std::cout<<rsakey<<std::endl;
	return true;
}



bool encrypt(const std::string& inMsg, std::string& outMsg, EVP_PKEY
*evpkey)
{
	char *in = (char*)malloc(inMsg.size()+1);
	size_t inlen = (size_t)inMsg.size();
	strcpy(in, inMsg.c_str());
	unsigned char *out; 
	size_t outlen; 

 	ctx = EVP_PKEY_CTX_new(evpkey,NULL);
	
	if (!ctx)
	{
		std::cout<<"error in pkey assign"<<std::endl;
	}     

 if (EVP_PKEY_encrypt_init(ctx) <= 0)
    {
		std::cout<<"EVP_PKEY_encrypt_init failure "<<std::endl;
	}   

 if (EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_PADDING) <= 0)
 {
	std::cout<<"error in padding"<<std::endl;
 }

 /* Determine buffer length */
 if (EVP_PKEY_encrypt(ctx, NULL, &outlen, (const unsigned char*)in, inlen)
<= 0)
 {
	std::cout<<"error in encrypt length"<<std::endl;
 }

 out = (unsigned char*)OPENSSL_malloc(outlen);

 if (!out)
 {
	std::cout<<" malloc failure "<<std::endl;
 } 

 if (EVP_PKEY_encrypt(ctx, out, &outlen, (const unsigned char*)in, inlen) <=
0)
 {
	std::cout<<"error in encrypt"<<std::endl;
 }

	std::cout<<"output length: "<<outlen<<std::endl;
	std::cout<<"output: "<<out<<std::endl;
	outMsg.assign((const char*)out, outlen);
}



bool decrypt(const std::string& inMsg, EVP_PKEY *evpkey)
{
	char *out = (char*)malloc(inMsg.size()+1);
	size_t outlen = (size_t)inMsg.size();
	strcpy(out, inMsg.c_str());

 ctx2 = EVP_PKEY_CTX_new(evpkey,NULL);
 if (!ctx2)
   {}     
 if (EVP_PKEY_decrypt_init(ctx2) <= 0)
     {}  
 if (EVP_PKEY_CTX_set_rsa_padding(ctx2, RSA_PKCS1_PADDING) <= 0)
       {} 

 /* Determine buffer length */
 if (EVP_PKEY_decrypt(ctx2, NULL, &outlen1, (const unsigned char*)out,
outlen) <= 0)
        {}

std::cout<<"dec output length: "<<outlen1<<std::endl;

 out1 = (unsigned char*)OPENSSL_malloc(outlen1);

 if (!out)
     {}   /* malloc failure */

 if (EVP_PKEY_decrypt(ctx2, out1, &outlen1, (const unsigned char*)out,
outlen) <= 0)
    {}    
out1[outlen1] = '\0';
std::cout<<"dec output length: "<<outlen1<<std::endl;
std::cout<<"dec output: "<<out1<<std::endl;

}


int main()
{
	RSA* rsamainkey = RSA_new();
	EVP_PKEY *key= EVP_PKEY_new();
	
	std::string in = "Hello encrypt this message test from main.";
	std::string ctext;

	GenerateRsaKeyPair();
	rsamainkey = rsakey;

	std::cout<<"main rsa key: "<<rsamainkey<<std::endl;

	if(!EVP_PKEY_assign_RSA(key, rsamainkey))
	std::cout<<"error in assign rsa."<<std::endl;

	std::cout<<"key: "<<key<<std::endl;

encrypt(in, ctext, key);

std::cout<<"ctext: "<<ctext<<std::endl;

decrypt(ctext, key);

}



--
View this message in context: http://openssl.6102.n7.nabble.com/Problem-in-RSA-encrypt-and-Decrypt-using-EVP-tp62759.html
Sent from the OpenSSL - User mailing list archive at Nabble.com.