Bug#730313: racoon and ipcomp not working for small packets

[Cacholong Info]

Hello,

Find attached the configuration files that I used for testing:

Host: ipsec1 (192.168.5.94):
 - ipsec1-ipsec-tools.conf
 - ipsec1.racoon.conf

Host: ipsec2 (192.168.5.95):
 - ipsec2-ipsec-tools.conf
 - ipsec2.racoon.conf

I will add the debug output of racoon later to this bug report.

Regards, Matthijs

#!/usr/sbin/setkey -f

# NOTE: Do not use this file if you use racoon with racoon-tool
# utility. racoon-tool will setup SAs and SPDs automatically using
# /etc/racoon/racoon-tool.conf configuration.
# 

# Flush the SAD and SPD
flush;
spdflush;

# AH
add 192.168.5.94 192.168.5.95 ah 15700 -A hmac-sha1 "12345678901234567890";
add 192.168.5.95 192.168.5.94 ah 24500 -A hmac-sha1 "12345678901234567890";

# ESP
add 192.168.5.94 192.168.5.95 esp 15701 -E rijndael-cbc 
"12345678901234567890123456789012";
add 192.168.5.95 192.168.5.94 esp 24501 -E rijndael-cbc 
"12345678901234567890123456789012";

# Add compression algo
add 192.168.5.94 192.168.5.95 ipcomp 0x100 -C deflate;
add 192.168.5.95 192.168.5.94 ipcomp 0x200 -C deflate;

# Security Policy (IPv4)
spdadd 192.168.5.94 192.168.5.95 any -P out ipsec
        ipcomp/transport//use
        esp/transport//require
        ah/transport//require;
spdadd 192.168.5.94 192.168.5.95 any -P in ipsec
        ipcomp/transport//use
        esp/transport//require
        ah/transport//require;


# Also read the Linux IPSEC Howto up at 
# http://www.ipsec-howto.org/t1.html 
log info;
path certificate "/etc/racoon/certs";

remote 192.168.5.95 {

        # Passive off, we will connect automatically
        passive off;

        # Allow only main mode
        exchange_mode main;

        # Verification (using x509 certificates)
        my_identifier asn1dn;
        peers_identifier asn1dn;
        verify_identifier on;
        verify_cert on;
        ca_type x509 "cacert.pem";
        certificate_type x509 "client.crt" "client.key";

        # Settings
        ike_frag on;
        nat_traversal off;
        generate_policy on;
        dpd_delay 10; # dead peer detection

        # Proposal
        proposal {
                encryption_algorithm aes 256;
                hash_algorithm sha1;
                authentication_method rsasig;
                dh_group 5;
        }
}

sainfo anonymous {
        encryption_algorithm aes 256;
        authentication_algorithm hmac_sha1;
        compression_algorithm deflate;
}


#!/usr/sbin/setkey -f

# NOTE: Do not use this file if you use racoon with racoon-tool
# utility. racoon-tool will setup SAs and SPDs automatically using
# /etc/racoon/racoon-tool.conf configuration.
# 

## Flush the SAD and SPD
#
flush;
spdflush;

# AH
add 192.168.5.94 192.168.5.95 ah 15700 -A hmac-sha1 "12345678901234567890";
add 192.168.5.95 192.168.5.94 ah 24500 -A hmac-sha1 "12345678901234567890";

# ESP
add 192.168.5.94 192.168.5.95 esp 15701 -E rijndael-cbc 
"12345678901234567890123456789012";
add 192.168.5.95 192.168.5.94 esp 24501 -E rijndael-cbc 
"12345678901234567890123456789012";

# Compression
add 192.168.5.94 192.168.5.95 ipcomp 0x100 -C deflate;
add 192.168.5.95 192.168.5.94 ipcomp 0x200 -C deflate;

# Security Policy (IPv4)
spdadd 192.168.5.95 192.168.5.94 any -P out ipsec
        ipcomp/transport//use
        esp/transport//require
        ah/transport//require;
spdadd 192.168.5.95 192.168.5.94 any -P in ipsec
        ipcomp/transport//use
        esp/transport//require
        ah/transport//require;


# Also read the Linux IPSEC Howto up at 
# http://www.ipsec-howto.org/t1.html 
log info;
path certificate "/etc/racoon/certs";

remote 192.168.5.94 {

        # Passive on, other end will connect automatically
        passive on;

        # Allow only main mode
        exchange_mode main;

        # Verification (using x509 certificates)
        my_identifier asn1dn;
        peers_identifier asn1dn;
        verify_identifier on;
        verify_cert on;
        ca_type x509 "cacert.pem";
        certificate_type x509 "client.crt" "client.key";

        # Settings
        ike_frag on;
        nat_traversal off;
        generate_policy on;
        dpd_delay 10; # dead peer detection

        # Proposal
        proposal {
                encryption_algorithm aes 256;
                hash_algorithm sha1;
                authentication_method rsasig;
                dh_group 5;
        }
}

sainfo anonymous {
        encryption_algorithm aes 256;
        authentication_algorithm hmac_sha1;
        compression_algorithm deflate;
}