RSA public and private key encrypt with fips issue

[Yi tiger]

Hi guys,

I have download openssl 1.0.2s and fips 2.0.16 and build successfully on 
windows and then I try to write a simple app encrypted with public key and 
decrypt with private key. But It failed decrypted with private key when fips 
mode is turned on, decrypt will be success if fips is turned off.
I got below error

[cid:image001.png@01D52176.09181B50]

The first error occurs if I pass RSA_sizeof(rsa) as  RSA_private_decrypt ‘s 
first parameter, the second error if I pass the encrypted buffer’s length as 
RSA_private_decrypt’s first parameter.
I also noticed return value of RSA_size is different when fips mode is turned 
on.
Its size is 256 if fips is turned off, but become 128 if fips turned on.
RSA_public_encrypt return value is equal to RSA_size when fips is turned off, 
but its return value will great than RSA_size when truned on, maybe this cause 
decrypt or something its my code problem?

Below is my source code,


#include "openssl\crypto.h"
#include "openssl\rsa.h"
#include "openssl\err.h"
#include "openssl/pem.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

char* privatekey;
int privlen;
char* publickey;
int publen;

void generate()
{
       int mode = FIPS_mode();
       if (mode == 0)
       {
              int ret = FIPS_mode_set(1);
              if (ret != 1)
              {
                     printf("error %s", ERR_get_error());
              }
       }

       RSA *rsa = RSA_generate_key(2048, 3, NULL, NULL);

       BIO *bio = BIO_new(BIO_s_mem());
       PEM_write_bio_RSAPrivateKey(bio, rsa, NULL, NULL, 0, NULL, NULL);
       int rszsize = RSA_size(rsa);
       privlen = BIO_pending(bio);
       privatekey = (char*)malloc(privlen + 1);
       BIO_read(bio, privatekey, privlen);

       PEM_write_bio_RSAPublicKey(bio, rsa);
       publen = BIO_pending(bio);
       publickey = (char*)malloc(publen + 1);
       BIO_read(bio, publickey, publen);

       BIO_free_all(bio);
       RSA_free(rsa);
}

void Encrypt(const char* password)
{
       BIO *bio = BIO_new(BIO_s_mem());
       RSA *rsa = NULL;
       BIO_write(bio, publickey, publen);
       RSA* temp = PEM_read_bio_RSAPublicKey(bio, &rsa, NULL, NULL);
       char encrypted[500];
       int len = 0;
       if (temp != NULL)
       {
              int rsasize = RSA_size(rsa);
              len = RSA_public_encrypt((int)strlen(password), (unsigned 
char*)password, (unsigned char*)encrypted, rsa, RSA_PKCS1_OAEP_PADDING);
              if (len == -1)
              {
                     int e = ERR_get_error();
                     char * err = ERR_error_string(ERR_get_error(), encrypted);
                     printf("RSA_public_encrypt failed.\n");
              }
              else
              {
                     printf("%s", encrypted);
              }
       }

       BIO *newbio = BIO_new_mem_buf(privatekey, privlen);
       RSA *privRsa = NULL;
       PEM_read_bio_RSAPrivateKey(newbio, &privRsa, NULL, NULL);

       char p[4096] = { 0 };
       int rsasize = RSA_size(privRsa);
       if (privRsa != NULL)
       {
              int ret = RSA_private_decrypt(rsasize, (unsigned char*)encrypted, 
(unsigned char*)p, privRsa, RSA_PKCS1_OAEP_PADDING);
              if (ret == -1)
              {
                     int e = ERR_get_error();
                     char * err = ERR_error_string(ERR_get_error(), p);
                     printf("error");
              }
       }
}

Thanks,
Tiger