Hello, I have installed OpenSSL 1.1.1c. I'm trying to make a custom OpenSSL engine for RSA. The following sample code is copied from the engine *e_dasync.c. *
Following is a sample code for my RSA engine (*rsa-engine.c*), =================================== */* Engine Id and Name */static const char *engine_rsa_id = "rsa-engine-new";static const char *engine_rsa_name = "RSA engine for testing";// data encryption functionstatic int eng_rsa_pub_enc(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { printf("RSA Engine is encrypting using public key\n"); return RSA_meth_get_pub_enc(RSA_PKCS1_OpenSSL()) (flen,from,to,rsa,padding);}// signature verifystatic int eng_rsa_pub_dec(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding) { printf("Signature verification\n"); return 0;}// signaturestatic int eng_rsa_priv_enc(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding){ printf("Signature method:\n"); return 0;}// data decryptionstatic int eng_rsa_priv_dec(int flen, const unsigned char *from, unsigned char *to, RSA *rsa, int padding){ printf("decryption method:\n"); return 0; }static RSA_METHOD *test_rsa_method = NULL;static int bind_dasync(ENGINE *e){ /* Setup RSA_METHOD */ if ((test_rsa_method = RSA_meth_new("Test RSA Engine", 0)) == NULL || RSA_meth_set_pub_enc(test_rsa_method, eng_rsa_pub_enc) == 0 || RSA_meth_set_pub_dec(test_rsa_method, eng_rsa_pub_dec) == 0 || RSA_meth_set_priv_enc(test_rsa_method, eng_rsa_priv_enc) == 0 || RSA_meth_set_priv_dec(test_rsa_method, eng_rsa_priv_dec) == 0 ) { return 0; } /* Ensure the dasync error handling is set up */ if (!ENGINE_set_id(e, engine_rsa_id) || !ENGINE_set_name(e, engine_rsa_name) || !ENGINE_set_RSA(e, test_rsa_method) ) { return 0; } return 1;}static int bind_helper(ENGINE *e, const char *id){ if (!bind_dasync(e)){ printf("2_Error: Inside Bind helper\n"); return 0; } return 1;}IMPLEMENT_DYNAMIC_BIND_FN(bind_helper)IMPLEMENT_DYNAMIC_CHECK_FN()* =================================== My *Makefile *looks like the following, =================================== *rsa-engine: gcc -g -fPIC -c rsa-engine.c gcc -g -shared -o librsa_engine.so -L./libdune rsa-engine.o -Bdynamic -lcrypto -lpthread mv librsa_engine.so rsa-engine-new.so sudo cp rsa-engine-new.so /opt/openssl/lib/engines-1.1/clean: rm -f *.o *.d *.so rsa-engine* =================================== My code compiles. When I try to do encryption using the following command, ========================= *openssl rsautl -encrypt -inkey public.pem -pubin -in msg.txt -out msg.enc -engine rsa-engine-new* ========================= I get a segmentation fault, ================================ *engine "rsa-engine-new" set.RSA Engine is encrypting using public keySegmentation fault (core dumped)* ================================ Do I need to Compile this sample engine with the OpenSSL in order for it to work? Regards, Shariful Alam