RSA digital signature verification failure with openssl 1.1.0j

[Ying <ydong@xxxxxxxxx>]

I am working on the upgrade my program written 4 years before with updated
openssl version 1.1.0j from 1.0.1e. There are so many changes between the
two versions. I updated my code with 1.1.0 API, but it failed and I cannot
figure out the reason. 

The RSA key pair and message signature are generated with openssl command
line as follows,

key generation and cert generation
$openssl req -new -keyout private/userkey.pem -out usercert-req.pem -config
./openssl.cnf
$openssl ca -in usercert-req.pem -out newcerts/usercert.pem -days 180
-config ./openssl.cnf

Sign.
openssl dgst -sha256 -sign userkey.pem -out signature.binary msg 
openssl dgst -hex -sha256 -sign userkey.pem -out signature.hex msg 


//get public key from cert
openssl x509 -pubkey -noout -in usercert.pem  > pubkey.pem

Verify
openssl dgst -sha256 -verify pubkey.pem -signature signature.binary msg

The result is verification OK

But I have to use the API to verify the signature in hex format, and the
result is verification failure. Can anyone help me to figure out the
problem.  The code first loads the certification and retrieve the public key
from certificate into EVP_PKEY, and then translate the hex digest (messasge)
to binary format, and use
EVP_DigestVerifyInitial/EVP_DigestVerifyUpdate/EVP_DigestVerifyFinal to
verify the signature.

It works in openssl 1.01e, and I tried to update the related functions in
the 1.1.0version, it still fails. Am I missing something in the new version?

This is the C code I am using:

#include <stdio.h>
#include <string.h>
#include <openssl/x509.h>
#include <openssl/pem.h>

#include <openssl/crypto.h>
#include <openssl/evp.h>
#include <openssl/engine.h>
#include <openssl/conf.h>
#include <openssl/comp.h>

#include <openssl/sha.h>
#include <openssl/objects.h>
#include <openssl/rsa.h>
#include <openssl/ssl.h>
#include <openssl/err.h>

int hex_to_bin(char ch)
{
	if ((ch >= '0') && (ch <= '9'))
		return ch - '0';
	if ((ch >= 'a') && (ch <= 'f'))
		return ch - 'a' + 0x0A;
	if ((ch >= 'A') && (ch <= 'F'))
		return ch - 'A' + 0x0A;
	return -1;
}


int hex2bin(unsigned char *dst, const char *src, unsigned count)
{
	while (count--) {
		int hi = hex_to_bin(*src++);
		int lo = hex_to_bin(*src++);

		if ((hi < 0) || (lo < 0))
			return -1;

		*dst++ = (hi << 4) | lo;
	}
	return 0;
}


int main() {

	FILE *fp;
	X509 *x509 = NULL;
	EVP_PKEY *pkey = NULL;
	unsigned char *sigbuf = NULL;
	unsigned char md[32];
	unsigned char *digest = NULL;
	
	unsigned int i, siglen = 0;

	siglen = 128;

	char line[siglen * 2 + 3];
	char *buf;
	int num = 0; //number of license


	const char cert_filestr[] = "./cert.pem";
	const char license_filename_str[] = "./signature";
	const char digest_filename_str[] = "./digest";

	//****************************************
	//load certificate and get pub key
	//*******************************************
	//OpenSSL_add_all_algorithms();
	//ERR_load_crypto_strings();


	fp = fopen(cert_filestr, "r");
	if (!fp) {
		printf("Error opening certificate file \n");
		return -1;
	}
	x509 = PEM_read_X509(fp, NULL, NULL, NULL);
	fclose(fp);

	pkey = X509_get_pubkey(x509);
	if(!pkey){
		printf("Fail to create pkey!");
		return -1;
	}	

	//*****************************************************
	//
	//load license.info (digest)
	//
	//****************************************************

	fp =fopen(digest_filename_str, "r");
	if(!fp)
	{	
	  printf("Error opening digest file\n");
	  return -1;
	}
	
	digest = (unsigned char*)malloc(64+10);
	digest = fgets(line, sizeof(line), fp);
	fclose(fp);



	//*************************************************
	//
	//load license.key (signature)
	//
	//
	//********************************************
	fp = fopen(license_filename_str, "r");
	if (!fp) {
		printf("Error opening license file \n");
		return -1;
	}

	//**********************************
	//change the sigature format from Hex to Binary
	//****************************************
	fp = fopen(license_filename_str, "r");

	unsigned int len = siglen ;
	sigbuf = (unsigned char*)malloc(len);
	

	if((buf = fgets(line, sizeof(line), fp)) != NULL) 
       {

		printf("*************Retrieved line of length %d :\n", (int)strlen(buf));
		printf("*************signature**********\n%s", buf);		
		hex2bin(sigbuf , buf, siglen);
		
	}
	fclose(fp);


	//**************************************************
	//
	//verify signature
	//
	//
	//******************************************************	

	EVP_MD_CTX *ctx = EVP_MD_CTX_create();
	i=0;
	if (!EVP_DigestVerifyInit(ctx, NULL, EVP_sha256(), NULL, pkey)) {

		printf("Error setting context\n");
		ERR_print_errors_fp(stderr);

		return -1;
	}

	if (!EVP_DigestVerifyUpdate(ctx, digest,67)) {
		ERR_print_errors_fp(stderr);
		return -1;
	}

	i = EVP_DigestVerifyFinal(ctx, sigbuf, 128);
	if (i <= 0) {
		printf("Verification Failure!\n");
		ERR_print_errors_fp(stderr);
		return -1;
	}


	X509_free(x509);
	free(sigbuf);
	

	return 0;

}






--
Sent from: http://openssl.6102.n7.nabble.com/OpenSSL-User-f3.html