d2i_RSAPrivateKey not working with compiler optmization 01

[Jayalakshmi bhat <bhat.jayalakshmi@xxxxxxxxx>]

Hi All, 

We upgraded our device to use OpenSSL 1.1.1k from OpenSSL 1.0.2h. Device is on an ARM processor. Embedded web server comes to ready state with compiler optimization set to -O0. 

With value -O1 we are seeing issues in d2i_RSAPrivateKey. 

I wrote a sample test program as below. The test program returns output of i2d_RSAPrivateKey as 1190 with optimization value -O0 and 1157 with optimization value -O1.

Has anyone faced this issue? Please let me know your thoughts and inputs.

Thanks and Regards,

Jayalakshmi

#define FALSE 0
#define ERROR -1
#define SSL_MAX_KEY_LEN 16342

#include <openssl/bio.h>
#include <openssl/evp.h>
#include <openssl/rsa.h>
#include <openssl/err.h>
#include <stdio.h>
#include <string.h>

int decodeBase64Data(char* base64Buf, char* decodeBuf, int dataLen, int bufLen, int  newlineFlag)
{
        BIO* bioBuf = NULL;      
        BIO* base64BioBuf = NULL;
        int returnVal = ERROR;   

        base64BioBuf = BIO_new(BIO_f_base64());
        if (base64BioBuf == NULL)
                return returnVal;

        if (newlineFlag == FALSE)
                BIO_set_flags(base64BioBuf, BIO_FLAGS_BASE64_NO_NL);

        bioBuf = BIO_new_mem_buf(base64Buf, dataLen);
        if (bioBuf == NULL)
                returnVal = ERROR;
        else
        {
                bioBuf = BIO_push(base64BioBuf, bioBuf);
                if (bioBuf != NULL)
                {
                        memset(decodeBuf, 0x0, bufLen);
                        dataLen = BIO_read(base64BioBuf,decodeBuf, bufLen);
                        returnVal = dataLen;
                } else  {
                        returnVal = ERROR;
                }
        }

        BIO_free_all(base64BioBuf);
        return returnVal;
}

 void testCertKey()

{

        int IDkeyLen;
        unsigned char *certBuf          = NULL;
        int certLen                             = 1624;
        int i;
        unsigned char *cert_copy        = NULL;
        EVP_PKEY *evp_key               = NULL;
        RSA *rsa                                   = NULL;
        unsigned char *IDkey               = (unsigned char*)malloc(SSL_MAX_KEY_LEN);
        certBuf                                      = (unsigned char*)malloc(SSL_MAX_KEY_LEN);
        strcpy(certBuf,"MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDJgN8Eq1gAd4ev8lq/Dwz68CBYWLyVQvQcAfTgR6PTsMCUL00q3NdEBXzkPEAG/15+IEEEypjO+SgJcmaHtsDk/oP7ggrOWu/7kj+x3RNcVotvOtJK1QdXUSViOgaldoJdtKSY2Y8RPFwcMWiqMM22JYCC7JDpSbAs4MwCiixTZogtwFiXQeCG3/v/qHt1MEQ4Fv821a9RVLwZ4MooSVy5KvIxt7qOZ+NKOBzin0FS/DF/79fPcbptg56Jv0J40l2xWgiPnRCmbz4TXrrChPGgXDhA2rUNZDLW+NUl7oRbqn4FwOWHGzn6a/27xiT+wmW+A1wRTi2V+u0AmFhGK79LAgMBAAECggEANFiPFwZwAQcML6/01QdS5ul0bZ73torDIXd8prrVm0uia5xyHn1qSphmXmAsfeTfJkK4wUbkNXhSZ0nZys22BIb9wFuC9SJ4j1DpzQnSWr180XNlZ905X/yLip3qVheYPEGONMrnuKGGiChpuTSebAAIB3rvFjt3UeE//jTo2O0moBuQZXWjsk8AcynTnJAcKJqSBtCCLPyS3dIVsGoaDCsAeZKQ5ZaTLxZ9vT3KbWrJi3+whU9Cx4HXRDe910Gq3E4cXjzhlDmxOPKVlGLX2cXWN4OhtRoXu7A3kUGxG/ZXxwYoXO8M5EoBA30LbUT7XpO9Lxzx+2hEBuYSHbA0qQKBgQDxYeptkl/XRP+vPxRRXtyANqm+iTpkOILv7PBMkEgJGOA8491GiylKpnF7KEmDhS2eSpsq7sDRg9ZuhPHMwkyouojUcVHvfNdXEbC2xYz0cIvzgqRCYhhWJGATdDq3Bt80GX6RX5hRO8EhJdFqP8uIG/cd2x2C113M70vshlBpHQKBgQDVtLip//7YGmTxOKhkZMr+9Csgct18LzKqkARciRtg2rlHTUveNvLcGXkev0JNzrDc8a1U2lGbFhXr8lo3s5zRCb+0rU3LNgZPaW+a9Qm6kJwL0uP0BNfL1Go+hZSrq1LVZdyo4oRHtiiA/WCqDOxrsQH5/gqEdLCH+AFbf5XFhwKBgAncGwxQv1N2wQ7yf6z3OACQuwxKZLU1TbvpLMOj3R1qA7bZA+/evVgMRnRmZ+JUbVGa8I2nZP7p6Ldo/MuFYEZAIFi1LEihC9bFpTdCCIjvk/flbTo88YuLPtVsmP1i7XkeVXowsnMXAsOM3On/IxwzMDvq+GCg1XcKi0plDiHNAoGATCMv63lGHU4CCqMJ84Kpg/4P42apHFZgcb4Pn7xFYdNrUUjrr8bZ7aqxqU+w2rG5922vOFzz2BIdS0uz8sZuJ1eW0lMwNScRRnelCIsmpS5iOiAvkL+s9nS+SXokBqlqQJci75SMVlRCx/n913f1P0T4FE37B0ffoQGt0iB5hr8CgYA5UpOkFs8rDu0xAQCqD3I2Yg35f77ATK2F3ZWdJOi7KnO67uz39QIYHao9iwJW1u6UXgAs18DAjmdd8EQSvr1ayhdSMfFnt/AWM0QbMIOrhytEd2KD7rPPeeZSFeCACFO9tPXasTVxPmiV0YhZueMAgwnmi3CLwm3pg8PF+3T5gQ==");

        IDkeyLen  = decodeBase64Data(certBuf, IDkey, certLen, SSL_MAX_KEY_LEN, FALSE);

        cert_copy = IDkey;
        evp_key = d2i_AutoPrivateKey(NULL,&cert_copy,IDkeyLen);
 

       if (evp_key == NULL)
        {
                printf("OpenSSL11k %d, %d, %s\n",IDkeyLen,ERR_get_error(),ERR_error_string(ERR_get_error(), NULL));
        }
        else
        {
                cert_copy = IDkey;
                rsa = EVP_PKEY_get1_RSA(evp_key); /*JEDI-11535 */
                IDkeyLen = i2d_RSAPrivateKey(rsa,&cert_copy);
                printf("OpenSSL11k: IDkeyLen %d %d \n",IDkeyLen,__LINE__);
        }      

}

int main()

{

     testCertKey();

}