RSA public/private keys only work when created programatically.

[David Henry]

I've written a bare bones enveloping example that takes a string, seals it
in an envelope, and then goes about opening it. Everything works just fine
if I generate my RSA keys programatically. Unfortunately, it does not work
if I encrypt the session keys with an RSA public key that was created on the
command line like:

> openssl genrsa -out rsaprivatekey.pem
> openssl rsa -in rsaprivatekey.pem -out rsapublickey.pem

I would greatly appreciate if someone could explain why my
programatically-created keys work, but the command-line ones do not. The
code that generates usable keys looks like this:

int main() {
    // generate & check keys ----
    RSA* rsa = RSA_generate_key(2048, RSA_F4, NULL, 0);
    int check_key = RSA_check_key(rsa);
    while (check_key <= 0) {
        cerr << "error generating keys -- regenerating...";
        rsa = RSA_generate_key(2048, RSA_F4, NULL, 0);
        check_key = RSA_check_key(rsa);
    }
    RSA_blinding_on(rsa, NULL);

    // write out pem-encoded public key ----
    BIO* rsaPublicBio = BIO_new_file("rsapublickey.pem", "w");
    PEM_write_bio_RSAPublicKey(rsaPublicBio, rsa);

    // write out pem-encoded encrypted private key ----
    BIO* rsaPrivateBio = BIO_new_file("rsaprivatekey.pem", "w");
    PEM_write_bio_RSAPrivateKey(rsaPrivateBio, rsa, NULL, NULL, 0, NULL,
NULL);

    BIO_free(rsaPublicBio);
    BIO_free(rsaPrivateBio);
    RSA_free(rsa);

    ...

    return 0;
}

The program that uses the keys is:

#include <cstdio>
#include <cstring>
#include <openssl/ssl.h>
#include <openssl/rand.h>
#include <openssl/ecdsa.h>

#define BUF_SIZE    4096
#define BLOCK_SIZE    32

int main() {
    // uninitialized symmetric cipher context ----
    EVP_CIPHER_CTX* ctx = new EVP_CIPHER_CTX;

    // symmetric cipher ----
    const EVP_CIPHER* type = EVP_aes_256_cbc();

    unsigned char
            message[BUF_SIZE] =

"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
    printf("Unencoded string = {%s}\n", message);

    int npubk = 1;
    unsigned char** ek = new unsigned char*[npubk];
    int* ekl = new int[npubk];
    EVP_PKEY** pubk = new EVP_PKEY*[npubk];

    // read in pem-encoded public key ----
    BIO* rsa_pub_bio = BIO_new_file("rsapublickey.pem", "r");
    RSA* rsa_pub = RSA_new();
    PEM_read_bio_RSAPublicKey(rsa_pub_bio, &rsa_pub, NULL, NULL);
    BIO_free(rsa_pub_bio);

    // encrypt symmetric session keys ----
    for (int i = 0; i < npubk; i++) {
        pubk[i] = EVP_PKEY_new();
        EVP_PKEY_assign_RSA(pubk[i], rsa_pub);
        ek[i] = new unsigned char[EVP_PKEY_size(pubk[i])];
        ekl[i] = EVP_PKEY_size(pubk[i]);
    }

    // random initialization vector ----
    unsigned char* iv = new unsigned char[EVP_MAX_IV_LENGTH];
    RAND_pseudo_bytes(iv, EVP_MAX_IV_LENGTH);

    int message_len;    // initialized by EVP_SealUpdate & EVP_SealFinal
    unsigned char encrypt_buf[BUF_SIZE + BLOCK_SIZE];

    EVP_SealInit(ctx, type, ek, &ekl[0], &iv[0], &pubk[0], npubk);
    // EVP_SealUpdate changes message_len to # bytes in message ----
    EVP_SealUpdate(ctx, encrypt_buf, &message_len, message, strlen((const
char*) message));
    printf("buf_len: %d\n", message_len);
    int total_len = message_len;    // line must be between SealUpdate &
SealFinal
    // EVP_SealFinal changes message_len value to # bytes of encryption
overhead ----
    EVP_SealFinal(ctx, &encrypt_buf[message_len], &message_len);

    int i;
    printf("Encoded string = {");
    for (i = 0; i < message_len; i++) {
        printf("%02x", encrypt_buf[i]);
    }

    for (i = 0; i < message_len; i++) {
        printf("%02x", encrypt_buf[i + total_len]);
    }
    printf("}\n");

    unsigned char decrypt_buf[BUF_SIZE];
    int decrypt_len;    // initialized by EVP_OpenUpdate & EVP_OpenFinal

    // read in pem-encoded encrypted private key ----
    BIO* rsa_priv_bio = BIO_new_file("rsaprivatekey.pem", "r");
    RSA* rsa_priv = RSA_new();
    PEM_read_bio_RSAPrivateKey(rsa_priv_bio, &rsa_priv, NULL, NULL);
    BIO_free(rsa_priv_bio);

    EVP_PKEY* privk = EVP_PKEY_new();
    EVP_PKEY_assign_RSA(privk, rsa_priv);

    EVP_OpenInit(ctx, type, *ek, ekl[0], &iv[0], privk);
    // EVP_OpenUpdate changes decrypt_len to # bytes in decrypted message
----
    EVP_OpenUpdate(ctx, decrypt_buf, &decrypt_len, encrypt_buf, total_len +
message_len);
    total_len = decrypt_len;    // line must be between OpenUpdate &
OpenFinal
    EVP_OpenFinal(ctx, &decrypt_buf[total_len], &decrypt_len);
    // EVP_OpenFinal changes decrypt_len value to # bytes of encryption
overhead ----
    decrypt_buf[total_len + decrypt_len] = '\0';

    printf("Unencoded string = {%s}\n", decrypt_buf);

    delete ctx;
    EVP_PKEY_free(privk);
    for (int i = 0; i < npubk; i++) {
        EVP_PKEY_free(pubk[i]);
        delete ek[i];
    }
    delete[] ek;
    delete[] ekl;
    delete[] pubk;
    delete[] iv;

    return 0;
}