PKCS7 encryption failed when processing concurrent large files (1.6G)

[vu le]

Dear all,

I wrote a function like this:


DLL_INT ECryptEncryptData(char* certFile, char* dataFile, char*
encryptedFile, char* errMsg, int errMsgLen)
{
    static char* func = "ECryptEncryptData";
    int rc = 0;
    char msg[MSG_LEN];
    BIO *in = NULL, *out = NULL;//, *tbio = NULL;//, *dout = NULL;
    BIO *bio_err_out = NULL;
    X509 *rcert = NULL;
    STACK_OF(X509) *recips = NULL;
    CMS_ContentInfo *cms = NULL;
    EVP_PKEY *publickey = NULL;

    EVP_CIPHER *eAlgt = NULL;
    int flags = CMS_BINARY;// | CMS_STREAM;// | CMS_DETACHED;
    if (errMsg == NULL || errMsgLen <= 0) {
        WriteLog(ERROR, "%s[%d] invalid input\n", func, __LINE__);
        return -1;
    }
    errMsg[0] = 0;
    printf_s("%s[%d] Infor dataFile[%s] encryptedFile[%s]\n", func,
__LINE__, dataFile, encryptedFile);

    //OpenSSL_add_all_algorithms();
    //ERR_load_crypto_strings();

        eAlgt = (EVP_CIPHER*)EVP_des_ede3_cbc();

    /* Read in recipient certificate */

    if ((publickey = getPublicKey(certFile, &rcert, msg, sizeof(msg), &rc))
== NULL) {
        sprintf_s(errMsg, errMsgLen, "%s[%d] getPublicKey fail %s\n", func,
__LINE__, msg);
        goto cleanup;
    }
    else {
        EVP_PKEY_free(publickey);
    }
    /* Create recipient STACK and add recipient cert to it */
    if ((recips = sk_X509_new_null()) == NULL || !sk_X509_push(recips,
rcert)) {
        sprintf_s(errMsg, errMsgLen, "%s[%d] sk_X509_new_null fail\n",
func, __LINE__);
        goto cleanup;
    }
    /* sk_X509_pop_free will free up recipient STACK and its contents
    * so set rcert to NULL so it isn't freed up twice.
    */
    rcert = NULL;

    /* Open content being encrypted */
    if ((in = BIO_new_file(dataFile, "r")) == NULL) {
        sprintf_s(errMsg, errMsgLen, "%s[%d] BIO_new_file fail\n", func,
__LINE__);
        goto cleanup;
    }
    /* encrypt content */
    else if ((cms = CMS_encrypt(recips, in, eAlgt, flags)) == NULL) {
        sprintf_s(errMsg, errMsgLen, "%s[%d] CMS_encrypt_WithHeader
fail\n", func, __LINE__);
        goto cleanup;
    }

    else if ((out = BIO_new_file(encryptedFile, "wb")) == NULL) {
        sprintf_s(errMsg, errMsgLen, "%s[%d] BIO_new_file fail\n", func,
__LINE__);
        goto cleanup;
    }
    /* Write out S/MIME message */
    else if (!i2d_CMS_bio_stream(out, cms, in, flags)) {
        sprintf_s(errMsg, errMsgLen, "%s[%d] i2d_CMS_bio_stream fail\n",
func, __LINE__);
        goto cleanup;
    }

cleanup:
    if (strlen(errMsg)) {
        WriteLog(ERROR, "%s", errMsg);
        if (!rc)
            rc = -1;
        bio_err_out = OpenLog();
    }
    if (cms)
        CMS_ContentInfo_free(cms);
    if (rcert)
        X509_free(rcert);
    if (recips)
        sk_X509_pop_free(recips, X509_free);
    if (in)
        BIO_free(in);
    if (out)
        BIO_free(out);
    /*if (dout)
    BIO_free(dout);*/
    /*if (tbio)
    BIO_free(tbio);*/
    if (bio_err_out) {
        if (MEM_LEAK_CHEK)
            CRYPTO_mem_leaks(bio_err_out);
        ERR_print_errors(bio_err_out);
        BIO_free(bio_err_out);
    }
    return rc;
}

I export it to an external dll for other application to call it.

Everything is OK for a single call to it. But when it be called concurrent
in multi threads (one thread proceeded a difference file), the return
encrypted file will fail (the file size some times has 0 bytes...).
I also try not use it in multi threads. I ran many applications at the same
time to encrypt multiple files (one application proceeded a difference
file) but it still failed.

I very much appreciate for any help
Thank you,

Vu Le