Hello, Starting from the rsa choice "rsa2048", there is crash due to segment fault when running speed. RSA512 and rsa1024 doesn't have this issue.
Following is the call stacks, registers info when crash happened. It shows that the register %rbp is zero at that point. The value of %rbp is set by instruction "movq %xmm2,$nptr " in label .L8x_no_tail. I don't understand how the algorithm in the x86_64-mont5.s works. So i have no idea why a zero value of %xmm2 was moved to %rbp at that point. Please help have a look. (wrdbg) bt #0 0x0000000000242d23 at __bn_sqrx8x_reduction+0x6c #1 0x000000000023f96b at bn_sqr8x_mont+0xcb #2 0x000000000023b014 at bn_mul_mont_fixed_top+0x64 #3 0x00000000002b61f5 at BN_mod_exp_mont+0x445 #4 0x0000000000281b1f at rsa_ossl_public_decrypt+0x28f #5 0x0000000000285c0a at int_rsa_verify+0x8a #6 0x0000000000285f92 at RSA_verify+0x32 #7 0x0000000000217963 at RSA_verify_loop+0x53 #8 0x00000000002176d8 at run_benchmark+0x108 #9 0x0000000000212e80 at speed_main+0x1400 #10 0x0000000000201411 in main (argc=1, argv=0x68ff20) at rtp.c:33 #11 0x0000000000201073 at _start+0x72 (wrdbg) info registers rax 0x0000000000000000 0 rdx 0x68f8f88932a43dc0 7564068842312580544 rcx 0xfffffffffffffff8 18446744073709551608 rbx 0x123959978a8b3640 1313179273754523200 rsi 0x0000000000000000 0 rdi 0x0000000000688430 6849584 rbp 0x0000000000000000 0 rsp 0x00000000006883b8 6849464 r8 0x123959978a8b3640 1313179273754523200 r9 0xf7a5b45905344009 17844867392957857801 r10 0x73dbc32a132b324b 8348480919738987083 r11 0x37cda4961a791379 4021051006907913081 r12 0x701103ae9d18bf8a 8075239655346847626 r13 0xbfbd1a5d2ab85477 13816228219293553783 r14 0x3f3610a53ecd4756 4554846375041124182 r15 0xea5e26bc1a343a39 16887978242021276217 rip 0x0000000000242d23 2370851 PC eflags 0x00000246 582 [ pf zf if ] fpu None xmm None __bn_sqrx8x_reduction: 0000000000242cb7: xor eax, eax 0000000000242cb9: mov rbx, qword ptr [rsp+0x28] 0000000000242cbe: mov rdx, qword ptr [rsp+0x38] 0000000000242cc3: lea rcx, ptr [rbp+r9*1-0x40] 0000000000242cc8: mov qword ptr [rsp+0x8], rcx 0000000000242ccd: mov qword ptr [rsp+0x10], rdi 0000000000242cd2: lea rdi, ptr [rsp+0x38] 0000000000242cd7: jmp 0x242ce0 <__bn_sqrx8x_reduction+41> 0000000000242cd9: nop dword ptr [rax], eax 0000000000242ce0: mov r9, qword ptr [rdi+0x8] 0000000000242ce4: mov r10, qword ptr [rdi+0x10] 0000000000242ce8: mov r11, qword ptr [rdi+0x18] 0000000000242cec: mov r12, qword ptr [rdi+0x20] 0000000000242cf0: mov r8, rdx 0000000000242cf3: imul rdx, rbx 0000000000242cf7: mov r13, qword ptr [rdi+0x28] 0000000000242cfb: mov r14, qword ptr [rdi+0x30] 0000000000242cff: mov r15, qword ptr [rdi+0x38] 0000000000242d03: mov qword ptr [rsp+0x20], rax 0000000000242d08: lea rdi, ptr [rdi+0x40] 0000000000242d0c: xor rsi, rsi 0000000000242d0f: mov rcx, -8 0000000000242d16: jmp 0x242d20 <__bn_sqrx8x_reduction+105> 0000000000242d18: nop dword ptr [rax+rax*1], eax 0000000000242d20: mov rbx, r8 0000000000242d23: mulx r8, rax, qword ptr [rbp] 0000000000242d29: adcx rax, rbx 0000000000242d2f: adox r8, r9 L8x_no_tail: adc 8*0($tptr),%r8 adc 8*1($tptr),%r9 adc 8*2($tptr),%r10 adc 8*3($tptr),%r11 adc 8*4($tptr),%r12 adc 8*5($tptr),%r13 adc 8*6($tptr),%r14 adc 8*7($tptr),%r15 adc \$0,%rax # top-most carry mov -8($nptr),%rcx # np[num-1] xor $carry,$carry movq %xmm2,$nptr # restore $nptr mov %r8,8*0($tptr) # store top 512 bits mov %r9,8*1($tptr) movq %xmm3,$num # $num is %r9, can't be moved upwards configdata.pm --dump: Command line (with current working directory = .): /usr/bin/perl ../Configure vxworks-x86-64 no-weak-ssl-ciphers no-aria no-blake2 no-camellia no-chacha no-cmac no-md2 no-md4 no-mdc2 no-ocb no-poly1305 no-rc2 no-rc5 no-scrypt no-seed no-siphash no-sm2 no-sm3 no-sm4 no-whirlpool no-autoerrinit no-comp no-sctp no-srtp --with-rand-seed=none -DOPENSSL_SYS_VXWORKS_SUPPORT -DVXWORKS_USER_SPACE -fPIC Perl information: /usr/bin/perl 5.22.1 for x86_64-linux-gnu-thread-multi Enabled features: asm async autoalginit autoload-config bf capieng cast cms ct deprecated des dgram dh dsa dso dtls dynamic-engine ec ec2m ecdh ecdsa engine err filenames gost hw(-.+)? idea multiblock nextprotoneg ocsp pic posix-io psk rc4 rdrand rfc3779 rmd160 shared sock srp sse2 ssl static-engine stdio tests tls ts ui-console tls1 tls1-method tls1_1 tls1_1-method tls1_2 tls1_2-method tls1_3 dtls1 dtls1-method dtls1_2 dtls1_2-method Disabled features: afalgeng [not-linux] aria [option] OPENSSL_NO_ARIA (skip crypto/aria) asan [default] OPENSSL_NO_ASAN autoerrinit [option] OPENSSL_NO_AUTOERRINIT blake2 [option] OPENSSL_NO_BLAKE2 (skip crypto/blake2) camellia [option] OPENSSL_NO_CAMELLIA (skip crypto/camellia) chacha [option] OPENSSL_NO_CHACHA (skip crypto/chacha) cmac [option] OPENSSL_NO_CMAC (skip crypto/cmac) comp [option] OPENSSL_NO_COMP (skip crypto/comp) crypto-mdebug [default] OPENSSL_NO_CRYPTO_MDEBUG crypto-mdebug-backtrace [default] OPENSSL_NO_CRYPTO_MDEBUG_BACKTRACE devcryptoeng [default] OPENSSL_NO_DEVCRYPTOENG ec_nistp_64_gcc_128 [default] OPENSSL_NO_EC_NISTP_64_GCC_128 egd [default] OPENSSL_NO_EGD external-tests [default] OPENSSL_NO_EXTERNAL_TESTS fuzz-libfuzzer [default] OPENSSL_NO_FUZZ_LIBFUZZER fuzz-afl [default] OPENSSL_NO_FUZZ_AFL heartbeats [default] OPENSSL_NO_HEARTBEATS makedepend [unavailable] md2 [option] OPENSSL_NO_MD2 (skip crypto/md2) md4 [option] OPENSSL_NO_MD4 (skip crypto/md4) mdc2 [option] OPENSSL_NO_MDC2 (skip crypto/mdc2) msan [default] OPENSSL_NO_MSAN ocb [option] OPENSSL_NO_OCB poly1305 [option] OPENSSL_NO_POLY1305 (skip crypto/poly1305) rc2 [option] OPENSSL_NO_RC2 (skip crypto/rc2) rc5 [option] OPENSSL_NO_RC5 (skip crypto/rc5) scrypt [option] OPENSSL_NO_SCRYPT sctp [option] OPENSSL_NO_SCTP seed [option] OPENSSL_NO_SEED (skip crypto/seed) siphash [option] OPENSSL_NO_SIPHASH (skip crypto/siphash) sm2 [option] OPENSSL_NO_SM2 (skip crypto/sm2) sm3 [option] OPENSSL_NO_SM3 (skip crypto/sm3) sm4 [option] OPENSSL_NO_SM4 (skip crypto/sm4) srtp [option] OPENSSL_NO_SRTP ssl-trace [default] OPENSSL_NO_SSL_TRACE threads [unavailable] ubsan [default] OPENSSL_NO_UBSAN unit-test [default] OPENSSL_NO_UNIT_TEST whirlpool [option] weak-ssl-ciphers [option] OPENSSL_NO_WEAK_SSL_CIPHERS zlib [default] zlib-dynamic [default] ssl3 [default] OPENSSL_NO_SSL3 ssl3-method [default] OPENSSL_NO_SSL3_METHOD Config target attributes: AR => "ar", ARFLAGS => "r", CC => "dummy", CFLAGS => "", HASHBANGPERL => "/usr/bin/env perl", RANLIB => "ranlib", RC => "windres", aes_asm_src => "aes-x86_64.s vpaes-x86_64.s bsaes-x86_64.s aesni-x86_64.s aesni-sha1-x86_64.s aesni-sha256-x86_64.s aesni-mb-x86_64.s", aes_obj => "aes-x86_64.o vpaes-x86_64.o bsaes-x86_64.o aesni-x86_64.o aesni-sha1-x86_64.o aesni-sha256-x86_64.o aesni-mb-x86_64.o", apps_aux_src => "", apps_init_src => "", apps_obj => "", bf_asm_src => "bf_enc.c", bf_obj => "bf_enc.o", bn_asm_src => "asm/x86_64-gcc.c x86_64-mont.s x86_64-mont5.s x86_64-gf2m.s rsaz_exp.c rsaz-x86_64.s rsaz-avx2.s", bn_obj => "asm/x86_64-gcc.o x86_64-mont.o x86_64-mont5.o x86_64-gf2m.o rsaz_exp.o rsaz-x86_64.o rsaz-avx2.o", bn_ops => "SIXTY_FOUR_BIT_LONG", build_file => "Makefile", build_scheme => [ "unified", "unix" ], cast_asm_src => "c_enc.c", cast_obj => "c_enc.o", cflags => "", chacha_asm_src => "chacha-x86_64.s", chacha_obj => "chacha-x86_64.o", cmll_asm_src => "cmll-x86_64.s cmll_misc.c", cmll_obj => "cmll-x86_64.o cmll_misc.o", cppflags => "\$(VSB_CONFIG_FILE_MACRO)", cpuid_asm_src => "x86_64cpuid.s", cpuid_obj => "x86_64cpuid.o", defines => [ ], des_asm_src => "des_enc.c fcrypt_b.c", des_obj => "des_enc.o fcrypt_b.o", disable => [ ], dso_extension => ".so", ec_asm_src => "ecp_nistz256.c ecp_nistz256-x86_64.s x25519-x86_64.s", ec_obj => "ecp_nistz256.o ecp_nistz256-x86_64.o x25519-x86_64.o", enable => [ ], exe_extension => "", includes => [ ], keccak1600_asm_src => "keccak1600-x86_64.s", keccak1600_obj => "keccak1600-x86_64.o", lflags => "", lib_cflags => "", lib_cppflags => "", lib_defines => [ ], md5_asm_src => "md5-x86_64.s", md5_obj => "md5-x86_64.o", modes_asm_src => "ghash-x86_64.s aesni-gcm-x86_64.s", modes_obj => "ghash-x86_64.o aesni-gcm-x86_64.o", module_cflags => "", module_cxxflags => "", module_ldflags => "-shared", padlock_asm_src => "e_padlock-x86_64.s", padlock_obj => "e_padlock-x86_64.o", poly1305_asm_src => "poly1305-x86_64.s", poly1305_obj => "poly1305-x86_64.o", rc4_asm_src => "rc4-x86_64.s rc4-md5-x86_64.s", rc4_obj => "rc4-x86_64.o rc4-md5-x86_64.o", rc5_asm_src => "rc5_enc.c", rc5_obj => "rc5_enc.o", rmd160_asm_src => "", rmd160_obj => "", sha1_asm_src => "sha1-x86_64.s sha256-x86_64.s sha512-x86_64.s sha1-mb-x86_64.s sha256-mb-x86_64.s", sha1_obj => "sha1-x86_64.o sha256-x86_64.o sha512-x86_64.o sha1-mb-x86_64.o sha256-mb-x86_64.o", shared_cflag => "", shared_defines => [ ], shared_extension => ".so.\$(SHLIB_VERSION_NUMBER)", shared_extension_simple => ".so", shared_ldflag => "-shared", shared_rcflag => "", shared_target => "dummy", sys_id => "VXWORKS", thread_defines => [ ], thread_scheme => "(unknown)", unistd => "<unistd.h>", uplink_aux_src => "", uplink_obj => "", wp_asm_src => "wp-x86_64.s", wp_obj => "wp-x86_64.o", Recorded environment: AR = BUILDFILE = CC = CROSS_COMPILE = CXX = HASHBANGPERL = OPENSSL_LOCAL_CONFIG_DIR = PERL = RANLIB = RC = WINDRES = __CNF_CFLAGS = __CNF_CPPDEFINES = __CNF_CPPFLAGS = __CNF_CPPINCLUDES = __CNF_CXXFLAGS = __CNF_LDFLAGS = __CNF_LDLIBS = Makevars: AR = ar ARFLAGS = r CC = dummy CFLAGS = -fPIC CPPDEFINES = OPENSSL_SYS_VXWORKS_SUPPORT VXWORKS_USER_SPACE CPPFLAGS = CPPINCLUDES = CXXFLAGS = -fPIC HASHBANGPERL = /usr/bin/env perl LDFLAGS = LDLIBS = PERL = /usr/bin/perl RANLIB = ranlib RC = windres NOTE: These variables only represent the configuration view. The build file template may have processed these variables further, please have a look at the build file for more exact data: Makefile build file: Makefile build file templates: ../Configurations/common0.tmpl ../Configurations/unix-Makefile.tmpl ../Configurations/common.tmpl Thanks, Wenbin