Bug in (possibly) BN or (probably) DH routines

[Lawrence MacIntyre]

Hi:

I have been having spurious problems with one of my client server
programs.  It uses a DH Key exchange to generate a blowfish key and
encrypts the data using that key.  I believe I have isolated the problem
to the minimum of code.  I'm attaching the test program, Makefile, and a
bash script that runs the program 1000 times.  It uses BN_new() and
BN_hex2bn() to convert two strings to BIGNUMs, stuffs them into a DH,
and then calls DH_generate_key() to generate the private/public key
pair.  When you run the program 1000 times, somewhere between 3 and 8
times the length of the public key will be 55 bytes instead of 56, as it
should be.  This breaks my client:-(  Once, the key was actually 54
bytes.  Am I doing something dopey or is this a bug?
-- 
                                 Lawrence
                                    ~
------------------------------------------------------------------------
Lawrence MacIntyre      Center for Information Infrastructure Technology
[EMAIL PROTECTED]   http://www.ciit.y12.doe.gov/~lpz     865.574.8696
# $Id: Makefile,v 1.2 2000/10/04 19:25:19 lpz Exp $
#
# Changes:
#CC = kgcc      #for RH 7.0
CC = gcc
CFLAGS = -g -Wall
COMMONOBJS =
INCLUDES =
INCLUDEDIRS = -I/usr/local/ssl/include
OBJS = dhtst.o
LIBDIRS = -L/usr/local/ssl/lib
LIBS = -lcrypto
RM = rm -f
EXES = dhtst
TAR = tar
ZIP = zip
SRCS = Makefile.dhtst dhtst.c dhtstloop.sh

all:    dhtst

dhtst:  dhtst.o
        $(CC) $(CFLAGS) -o dhtst dhtst.o $(COMMONOBJS) $(LIBDIRS) $(LIBS)

.PHONY: clean tar zip

clean:
        $(RM) $(OBJS) $(COMMONOBJS) $(EXES) *.tar *.zip

tar:
        $(TAR) czvf openssltst.tgz $(SRCS)

zip:
        $(ZIP) openssltst.zip $(SRCS)
          
dhtst.o:        dhtst.c Makefile.dhtst
        $(CC) $(CFLAGS) -c $(INCLUDEDIRS) dhtst.c

dhtstloop.sh

#include <stdio.h>
#include <string.h>

#include <openssl/bn.h>
#include <openssl/dh.h>
#include <openssl/err.h>

int main(int argc, char **argv)
{
        int status;
        DH *a = NULL;
        char p1[113];
        char g[3];              

        memcpy(p1,
                                
"CA9C3CB3E239845076ACC3963634A02F1A5003209B29B1BF317E18A0D2440A630825C0C3E3F7225859629117C7DF2899493C7C49B10F8937",
                                112);
        p1[112] = '\0';

        memcpy(g, "05", 2);
        g[2] = '\0';

        a = DH_new();
        if(a == NULL) {
                perror("DH_new(a): ");
                status = ERR_get_error();
                goto err;
        }
        
        a->p = BN_new();
        if(a->p == NULL) {
                status = ERR_get_error();
                goto err;
        }
        
        status = BN_hex2bn(&(a->p), p1);
        if(status != 112) {
                printf("P is bogus\n");
          if(status == 0) {
                  status = ERR_get_error();
                  goto err;
                }
        }

        a->g = BN_new();
        if(a->g == NULL) {              
                status = ERR_get_error();
                goto err;
        }
        
        status = BN_hex2bn(&(a->g), g);
        if(status != 2) {
                printf("G is bogus\n");
        if(status == 0) {
                  status = ERR_get_error();
                  goto err;
                }
        }
                
        if(!DH_generate_key(a)) {
                perror("DH_generate key: ");
                status = ERR_get_error();
                goto err;
        }
        status = BN_num_bytes(a->pub_key);
        if(status != 56) {
        printf("local pub key bytes: %d\n", status);
        }
        
        status = 0;
err:
        if(status != 0) {
                printf("Status: %d\n", status);
        }
  return(status);
}