PKI encryption failing on 32-bit Solaris

["Perrow, Graeme" <graeme.perrow@xxxxxxx>]

I have a small test program (source attached) that does a very simple PKI encrypt / decrypt. This program works on Windows, Linux, and Solaris (64-bit) but fails if I run a 32-bit version on Solaris 10. Solaris 11 is fine.

 

If I use “./config -kPIC -m32 -xarch=sparc” to build OpenSSL, I get a crash in bn_mul_mont_t4_32. I added “no-asm” and it no longer crashes but I get this error output:

 

OSSL error

4275158204:error:0407109F:rsa routines:RSA_padding_check_PKCS1_type_2:pkcs decoding error:rsa_pk1.c:272:

 

End OSSL error

decrypt failed

 

I also tried adding -d to the config line and then the program succeeds though is very verbose.

 

Could this be a bug in OpenSSL, or am I configuring it incorrectly, or is there a bug in my code?

 

Thanks for any insight

Graeme Perrow

 

#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#include "openssl/rsa.h"
#include "openssl/pem.h"
#include "openssl/err.h"
#include "openssl/ssl.h"
#include "openssl/rand.h"

int _ossl_err_printf_callback( const char *str, size_t len, void *u )
/*******************************************************************/
{
    printf( "%s\n", str );
    return 0;
}

void _ossl_err_printf( void )
/***************************/
{
    if( ERR_peek_error() > 0 ) {
        printf( "OSSL error\n" );
        ERR_print_errors_cb( _ossl_err_printf_callback, NULL );
        printf( "End OSSL error\n" );
    }
}

const char *PUBLIC_KEY = "-----BEGIN RSA PUBLIC KEY-----\n"
    "MIIBCgKCAQEAvg95SwHiiiN/ttddVS7nUR1Gtbg1xMgFdrPUTidgGmS2DM5k/y7B\n"
    "VNCIp93gLjg5fKB3nqcuIhqppEDadDdWhlRFjImRqUlhppYjoIuP5t7tXHqzTwa8\n"
    "QUHZ29Y8/CZXwtpud4C7o2vzJStFVXc1goehTD1lig9KXI5FAZDyKsGlzJPNm0+c\n"
    "YrU7UNXA6DtRdKhVNynHuCRaoglkZ/5x4Qxk3O4yCeuuttTqcfo8hqOiM8TtQGgD\n"
    "sOlOHgn26abs+rpAtgqyQIVogRVfQKm1pfatqK/PvVdw75/c9t88tglzPhTo3CHH\n"
    "e5wv67m08Wy5TKiQL+SYZi9wQA/ktdUc5wIDAQAB\n"
    "-----END RSA PUBLIC KEY-----";
const char *PRIVATE_KEY = "-----BEGIN RSA PRIVATE KEY-----\n"
    "MIIEoQIBAAKCAQEAvg95SwHiiiN/ttddVS7nUR1Gtbg1xMgFdrPUTidgGmS2DM5k\n"
    "/y7BVNCIp93gLjg5fKB3nqcuIhqppEDadDdWhlRFjImRqUlhppYjoIuP5t7tXHqz\n"
    "Twa8QUHZ29Y8/CZXwtpud4C7o2vzJStFVXc1goehTD1lig9KXI5FAZDyKsGlzJPN\n"
    "m0+cYrU7UNXA6DtRdKhVNynHuCRaoglkZ/5x4Qxk3O4yCeuuttTqcfo8hqOiM8Tt\n"
    "QGgDsOlOHgn26abs+rpAtgqyQIVogRVfQKm1pfatqK/PvVdw75/c9t88tglzPhTo\n"
    "3CHHe5wv67m08Wy5TKiQL+SYZi9wQA/ktdUc5wIDAQABAoH/QtEt0zDWc7oHLECK\n"
    "j2ft6pZgRYJ9ZLDtOs391XwCTUUau2XUG+JUFX06jclzBsy9aRMA+sXLcjHtcdFa\n"
    "+4B46sC1srKqgtlJe0dNPyPRc1jsGZcJcr7rcAW2kZK/3eWc9OPV8i1PyiQpVuff\n"
    "eCxwo3MLQcJT8DVv7xoVKHfMqzrNt6hbymGMklpw9Nwc/CqGq28jw1K1+h1Ekryt\n"
    "hwqr4C0bDvIb/mJ+ipVbakOSFytNaVJeyB2vfS8sJ/2ojZieraloiLHSY6vCZPGC\n"
    "w+DMHUIAHVCfHN23arZStHxAlqWjpgECRWB9Y5oHCRtZDcUx6eCfn9IIkTX4TdVK\n"
    "LDhRAoGBAOleqXQM7nlOYavwWaClR75mVhT1wCf9hgq9hhlJbWS67xwWCAIQVp2x\n"
    "G3M/iNPbDUBqfIsrS7h3lox5p/QijmBdwdchvc4a8EETr+0XSe/+E7p11RoWVBzd\n"
    "LmUoAcM4yU/S5ro/WJ6+iyk1y3og0+TJ50Cx0Tk8Khpj7AdhL5GVAoGBANB9q45T\n"
    "YwPsyCVAVvMxFxk2QboBhLh5sYbTbqsT/JMSowav/09vpL6+ZGvSz1DoD5NVusgw\n"
    "LAgUt0ZczqCE8xRdl3RmfxRDeElgBQE2fXYIcNiKIpJNcr9bLfDusu3myAYiPWEH\n"
    "xv68zmtMTdmlmB9dkkWIMMome0YsWDhWCo2LAoGAFFuKorICF5jr0kjsOKRSZtyc\n"
    "g8WQVqy6BbPUv9306gWYmDWfyzZLqAPsJhiTwg9oz4a6iJVlTJk6C9UM/rIJA4UK\n"
    "RM8PtKwK61JhVWhjrcjXRbvDtDvgaIgu5OohhTst7IW0VOjzvR1YHGcx8dxtP5cR\n"
    "UkZwvvJtpdGzpE0pDiUCgYEAj/pZIN6BAFLiHKeD5EnAU87eMN+fke2oJR2ZzVbu\n"
    "E3rJTnL38xl8QaUloDhiptK6/tozNM/feG0l69dncf6eBzBiySWHGK1HQhSnICFL\n"
    "HtnRZBhwq2wElBUZcsrEkfnPpDy8+mbuizhDrGFzWZw7o11xB8d3OgK9GsbA3Za9\n"
    "juUCgYA5t2LtfTEMye2YJs6IHb8cD3LlUAN8RO6Wju63PtDUfjuiBc9z8XcDkaBa\n"
    "GCW5y6oKfchFUo6IN8VMY8LjsRX6rXsfvc8MPi6f0Z+CLL1/AqM27Wdn6SvhEy89\n"
    "5q5blS0A+fTdcHPCGle5YROBWSEF5hbwl2E9AEkq8E4MbVPF+g==\n"
    "-----END RSA PRIVATE KEY-----";

int main( int, char ** )
/**********************/
{
    OpenSSL_add_all_algorithms();
    RAND_poll();
    SSL_load_error_strings();

    BIO *bio = BIO_new_mem_buf( (void*)PUBLIC_KEY, (int)strlen(PUBLIC_KEY) );
    char *		name = NULL;
    char *		header2 = NULL;
    unsigned char *	data = NULL;
    long		len;
    RSA *rsa;
    EVP_PKEY *key;
    size_t keylen;
    int rsa_pad;
    char *input;
    size_t input_len;
    char * output;
    size_t output_len;
    char *decrypted;

    int rc = PEM_read_bio( bio, &name, &header2, &data, &len );
    if( rc != 1 ) {
	fprintf( stderr, "PEM_read_bio returned %d\n", rc );
	return 1;
    }
    BIO_free( bio );

    bio = BIO_new_mem_buf( (void*)data, (int)len );
    rsa = d2i_RSAPublicKey_bio( bio, NULL );
    if( rsa == NULL ) {
	fprintf( stderr, "d2i_RSAPublicKey_bio failed\n" );
	return 1;
    }

    key = EVP_PKEY_new();
    EVP_PKEY_assign_RSA( key, rsa );
    BIO_free( bio );

    keylen = EVP_PKEY_size( key );
    
    rsa_pad = RSA_PKCS1_PADDING;

    input = new char[keylen-11];
    input_len = keylen-11;
    memset( input, 'a', keylen-11 );

    output = new char[keylen];
    output_len = RSA_public_encrypt( (int)input_len, (unsigned char *)input,
					    (unsigned char *)output, rsa,
					    rsa_pad );
    if( output_len == -1 ) {
	_ossl_err_printf();
	fprintf( stderr, "public_encrypt failed\n" );
	return 1;
    }

    // DECRYPT
    bio = BIO_new_mem_buf( (void*)PRIVATE_KEY, (int)strlen(PRIVATE_KEY) );
    rsa = PEM_read_bio_RSAPrivateKey( bio, NULL, NULL, NULL );
    key = EVP_PKEY_new();
    EVP_PKEY_assign_RSA( key, rsa );
    BIO_free( bio );
    keylen = EVP_PKEY_size( key );

    decrypted = new char[keylen];
    output_len = RSA_private_decrypt( (int)output_len, (unsigned char *)output,
				      (unsigned char *)decrypted,
				      rsa, rsa_pad );
    if( output_len == -1 ) {
	_ossl_err_printf();
	fprintf( stderr, "decrypt failed\n" );
	return 1;
    }
    decrypted[output_len] = '\0';
    printf( "Decrypted: %s\n", decrypted );

    RSA_free( rsa );
    delete [] input;
    delete [] output;
    delete [] decrypted;

    return 0;
}

-- 
openssl-users mailing list
To unsubscribe: https://mta.openssl.org/mailman/listinfo/openssl-users