I am new to making engines in openssl. Basically I want to implement an OpenSSL RSA engine that uses the functions I mentioned while encrypting and decrypting. My engine compiles and loads but it seems that it is not using the functions I want it to use
for encryption and decryption.
#include <openssl/opensslconf.h>
#include <stdio.h>
#include <string.h>
#include <openssl/crypto.h>
#include <openssl/buffer.h>
#include <openssl/engine.h>
#include <openssl/rsa.h>
#include <openssl/bn.h>
#include <openssl/err.h>
static int
eng_rsa_pub_enc (int flen, const unsigned char *from,
unsigned char *to, RSA * rsa, int padding)
{
printf ("Engine is encrypting using pub key \n");
RSA_public_encrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}
static int
eng_rsa_pub_dec (int flen, const unsigned char *from,
unsigned char *to, RSA * rsa, int padding)
{
printf ("Engine is decrypting using pub key \n");
RSA_public_decrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}
static int
eng_rsa_priv_enc (int flen, const unsigned char *from, unsigned char *to,
RSA * rsa, int padding __attribute__ ((unused)))
{
printf ("Engine is encrypting using priv key \n");
RSA_private_encrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}
static int
eng_rsa_priv_dec (int flen, const unsigned char *from, unsigned char *to,
RSA * rsa, int padding __attribute__ ((unused)))
{
printf ("Engine is decrypting using priv key \n");
RSA_private_decrypt (flen, from, to, rsa, RSA_PKCS1_PADDING);
}
/* Constants used when creating the ENGINE */
static const char *engine_rsa_id = "rsa-engine 1";
static const char *engine_rsa_name = "engine for testing 1";
static RSA_METHOD struct_rsa = {
"RSA engine for demo",
eng_rsa_pub_enc,
eng_rsa_pub_dec,
eng_rsa_priv_enc,
eng_rsa_priv_dec,
NULL,
NULL,
NULL,
NULL,
RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE,
NULL,
NULL,
NULL
};
static int bind (ENGINE * e, const char *id)
{
printf ("%s\n", id);
if (!ENGINE_set_id (e, engine_rsa_id) ||
!ENGINE_set_name (e, engine_rsa_name) ||
!ENGINE_set_RSA (e, &struct_rsa))
return 0;
return 1;
}
IMPLEMENT_DYNAMIC_BIND_FN (bind)
IMPLEMENT_DYNAMIC_CHECK_FN ()
I am compiling the code using following command.
gcc -fPIC -c rsa-engine.c
gcc -shared -o librsa_engine.so -lcrypto rsa-engine.o
openssl engine -t -c rsa_engine
Here the engine loads but when i try to encrypt a text file by using following command
openssl pkeyutl -encrypt -in message.txt -pubin -inkey pubkey-B.pem -engine rsa_engine -out cipher.bin
openssl pkeyutl -decrypt -in cipher.bin -inkey privkey-B.pem -engine rsa_engine -out rec.txt
It seems that it is not using the functions which I defined in the struct_rsa. It is also not giving the output from the printf in the function.
