Added an IPv4 and MAC addresses management system to ovn-northd. When a logical switch's options:subnet field is set, logical ports attached to that switch that do not have a MAC/IPv4 address will automatically be allocated a globally unique MAC address/unused IPv4 address within the provided subnet. This can be useful for a user who wants to deploy many VM's or containers with networking capabilities, but does not care about the specific MAC/IPv4 addresses that are assigned.
Added tests in ovn.at for ipam. Signed-off-by: Nimay Desai <nimaydes...@gmail.com> --- AUTHORS | 1 + ovn/northd/ovn-northd.c | 342 ++++++++++++++++++++++++++++++++++++++++++++++++ ovn/ovn-nb.ovsschema | 7 +- ovn/ovn-nb.xml | 14 +- tests/ovn.at | 156 ++++++++++++++++++++++ 5 files changed, 517 insertions(+), 3 deletions(-) diff --git a/AUTHORS b/AUTHORS index 704ba40..ce4501a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -164,6 +164,7 @@ Murphy McCauley murphy.mccau...@gmail.com Natasha Gude nata...@nicira.com Neil McKee neil.mc...@inmon.com Neil Zhu z...@centecnetworks.com +Nimay Desai nimaydes...@gmail.com Nithin Raju nit...@vmware.com Niti Rohilla niti.rohi...@tcs.com Numan Siddique nusid...@redhat.com diff --git a/ovn/northd/ovn-northd.c b/ovn/northd/ovn-northd.c index 4fa4f7c..1f47c65 100644 --- a/ovn/northd/ovn-northd.c +++ b/ovn/northd/ovn-northd.c @@ -56,6 +56,12 @@ static const char *ovnsb_db; static const char *default_nb_db(void); static const char *default_sb_db(void); + +#define MAC_ADDR_PREFIX 0x0A0000000000L +#define MAC_ADDR_SPACE 0xffffff +/* MAC address table of "struct eth_addr"s, that holds the MAC addresses + * allocated by the ipam. */ +static struct hmap macam = HMAP_INITIALIZER(&macam); /* Pipeline stages. */ @@ -261,8 +267,40 @@ struct ovn_datapath { uint32_t port_key_hint; bool has_unknown; + + /* IPAM data. */ + struct hmap ipam; +}; + +struct macam_node { + struct hmap_node hmap_node; + struct eth_addr mac_addr; /* Allocated MAC address. */ +}; + +static void +cleanup_macam(struct hmap *macam) +{ + struct macam_node *node; + HMAP_FOR_EACH_POP (node, hmap_node, macam) { + free(node); + } +} + +struct ipam_node { + struct hmap_node hmap_node; + uint32_t ip_addr; /* Allocated IP address. */ }; +static void +destroy_ipam(struct hmap *ipam) +{ + struct ipam_node *node; + HMAP_FOR_EACH_POP (node, hmap_node, ipam) { + free(node); + } + hmap_destroy(ipam); +} + static struct ovn_datapath * ovn_datapath_create(struct hmap *datapaths, const struct uuid *key, const struct nbrec_logical_switch *nbs, @@ -275,6 +313,7 @@ ovn_datapath_create(struct hmap *datapaths, const struct uuid *key, od->nbs = nbs; od->nbr = nbr; hmap_init(&od->port_tnlids); + hmap_init(&od->ipam); od->port_key_hint = 0; hmap_insert(datapaths, &od->key_node, uuid_hash(&od->key)); return od; @@ -289,6 +328,7 @@ ovn_datapath_destroy(struct hmap *datapaths, struct ovn_datapath *od) * use it. */ hmap_remove(datapaths, &od->key_node); destroy_tnlids(&od->port_tnlids); + destroy_ipam(&od->ipam); free(od->router_ports); free(od); } @@ -557,6 +597,303 @@ ovn_port_allocate_key(struct ovn_datapath *od) } static void +ipam_insert_mac(struct eth_addr *ea, bool check) +{ + if (!ea) { + return; + } + + uint64_t mac64 = eth_addr_to_uint64(*ea); + if (check) { + /* Checking for duplicate MAC addresses. */ + struct macam_node *macam_node; + HMAP_FOR_EACH_WITH_HASH (macam_node, hmap_node, hash_uint64(mac64), + &macam) { + if (eth_addr_equals(*ea, macam_node->mac_addr)) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1); + VLOG_WARN_RL(&rl, "Duplicate MAC set: "ETH_ADDR_FMT, + ETH_ADDR_ARGS(macam_node->mac_addr)); + return; + } + } + } + + /* Insert new MAC into MACAM if it is not a duplicate and was + * assigned by this address management system. */ + if (!((mac64 ^ MAC_ADDR_PREFIX) >> 24)) { + struct macam_node *new_macam_node = xmalloc(sizeof *new_macam_node); + new_macam_node->mac_addr = *ea; + hmap_insert(&macam, &new_macam_node->hmap_node, hash_uint64(mac64)); + } +} + +static void +ipam_insert_ip(struct ovn_datapath *od, uint32_t ip, bool check) +{ + if (!od) { + return; + } + + if (check) { + /* Checking for duplicate IP addresses. */ + struct ipam_node *ipam_node; + HMAP_FOR_EACH_WITH_HASH (ipam_node, hmap_node, hash_int(ip, 0), + &od->ipam) { + if (ipam_node->ip_addr == ip) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1); + VLOG_WARN_RL(&rl, "Duplicate IP set: "IP_FMT, + IP_ARGS(htonl(ip))); + return; + } + } + } + + struct ipam_node *new_ipam_node = xmalloc(sizeof *new_ipam_node); + new_ipam_node->ip_addr = ip; + hmap_insert(&od->ipam, &new_ipam_node->hmap_node, hash_int(ip, 0)); +} + +static void +ipam_add_port_addresses(struct ovn_datapath *od, struct ovn_port *op) +{ + if (!od || !op) { + return; + } + + if (op->nbs) { + /* Add all the port's addresses to address data structures. */ + for (size_t i = 0; i < op->nbs->n_addresses; i++) { + struct lport_addresses laddrs; + if (!extract_lsp_addresses(op->nbs->addresses[i], &laddrs, + false)) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1); + VLOG_WARN_RL(&rl, "Extract addresses failed."); + continue; + } + ipam_insert_mac(&laddrs.ea, true); + + /* IP is only added to IPAM if the switch's subnet option + * is set, whereas MAC is always added to MACAM. */ + if (!smap_get(&od->nbs->options, "subnet")) { + continue; + } + + for (size_t j = 0; j < laddrs.n_ipv4_addrs; j++) { + uint32_t ip = ntohl(laddrs.ipv4_addrs[j].addr); + ipam_insert_ip(od, ip, true); + } + } + } else if (op->nbr) { + struct eth_addr mac; + if (!eth_addr_from_string(op->nbr->mac, &mac)) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); + VLOG_WARN_RL(&rl, "bad 'mac' %s", op->nbr->mac); + return; + } + ipam_insert_mac(&mac, true); + + if (!op->peer || !op->peer->nbs + || !smap_get(&op->peer->nbs->options, "subnet")) { + return; + } + + ovs_be32 ip, mask; + char* error = ip_parse_masked(op->nbr->network, &ip, &mask); + if (error || mask == OVS_BE32_MAX || !ip_is_cidr(mask)) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); + VLOG_WARN_RL(&rl, "bad 'network' %s", op->nbr->network); + free(error); + return; + } + ipam_insert_ip(od, ntohl(ip), true); + } +} + +static uint64_t +ipam_get_unused_mac(void) +{ + struct macam_node *macam_node; + uint64_t mac64; + struct eth_addr mac; + bool used; + for (uint32_t i = 1; i < MAC_ADDR_SPACE; i++) { + mac64 = MAC_ADDR_PREFIX | i; + eth_addr_from_uint64(mac64, &mac); + used = false; + HMAP_FOR_EACH_WITH_HASH (macam_node, hmap_node, + hash_uint64(mac64), + &macam) { + if (eth_addr_equals(mac, macam_node->mac_addr)) { + used = true; + break; + } + } + if (!used) { + break; + } + } + + if (mac64 == MAC_ADDR_SPACE) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); + VLOG_WARN_RL(&rl, "MAC address space exhausted."); + mac64 = 0; + } + + return mac64; +} + +static uint32_t +ipam_get_unused_ip(struct ovn_datapath *od, uint32_t subnet, uint32_t mask) +{ + if (!od) { + return 0; + } + + struct ipam_node *ipam_node; + uint32_t ip = 0; + + /* Find first unused IP address in subnet. x.x.x.1 is reserved for + * a logical router port. */ + bool used; + for (uint32_t i = 2; i < ~mask; i++) { + uint32_t tentative_ip = subnet + i; + used = false; + /* Check if IP already exists in IPAM records. */ + HMAP_FOR_EACH_WITH_HASH (ipam_node, hmap_node, + hash_int(tentative_ip, 0), + &od->ipam) { + if (ipam_node->ip_addr == tentative_ip) { + used = true; + break; + } + } + if (!used) { + ip = tentative_ip; + break; + } + } + if (!ip) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 1); + VLOG_WARN_RL( &rl, "Subnet address space has been exhausted."); + } + + return ip; +} + +static void +ipam_allocate_addresses(struct ovn_datapath *od, struct ovn_port *op, + ovs_be32 subnet, ovs_be32 mask) +{ + if (!od || !op || !op->nbs) { + return; + } + + /* Allocate MAC address. */ + struct eth_addr mac; + if (!op->nbs->n_addresses) { + uint64_t mac64 = ipam_get_unused_mac(); + eth_addr_from_uint64(mac64, &mac); + } else { + /* If op already has at least one IPv4 address, do not allocate any + * addresses for it. Otherwise, use one of op's MAC addresses that + * does not have an IPv4 address associated with it to construct a new + * address. */ + struct lport_addresses laddrs; + for (size_t i = 0; i < op->nbs->n_addresses; i++) { + if (!extract_lsp_addresses(op->nbs->addresses[i], &laddrs, + false)) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1); + VLOG_INFO_RL(&rl, "invalid syntax '%s' in addresses. No MAC" + " address found", op->nbs->addresses[i]); + return; + } else if (laddrs.n_ipv4_addrs > 0) { + return; + } else { + mac = laddrs.ea; + } + } + } + + /* Allocate ip and add it to IPAM hmap. */ + uint32_t ip = ipam_get_unused_ip(od, ntohl(subnet), ntohl(mask)); + if (!ip) { + return; + } + ipam_insert_ip(od, ip, false); + + /* Add MAC to MACAM hmap if it is newly allocated and an IPv4 address was + * successfully allocated. */ + if (!op->nbs->n_addresses) { + ipam_insert_mac(&mac, false); + } + + /* Convert MAC to string form. */ + struct ds mac_ds; + ds_init(&mac_ds); + ds_put_format(&mac_ds, ETH_ADDR_FMT, ETH_ADDR_ARGS(mac)); + + /* Convert IP to string form. */ + struct ds ip_ds; + ds_init(&ip_ds); + ds_put_format(&ip_ds, IP_FMT, IP_ARGS(htonl(ip))); + + char *new_addr = xasprintf("%s %s", mac_ds.string, ip_ds.string); + nbrec_logical_switch_port_set_addresses(op->nbs, (const char **) &new_addr, + 1); + ds_destroy(&mac_ds); + ds_destroy(&ip_ds); +} + +static void +build_ipam(struct northd_context *ctx, struct hmap *datapaths, + struct hmap *ports) +{ + if (!ctx->ovnnb_txn) { + return; + } + + /* If the switch's options:subnet is set, allocate new addresses for ports + * that do not have any. */ + struct ovn_datapath *od; + HMAP_FOR_EACH (od, key_node, datapaths) { + if (od->nbs) { + const char *subnet_str = smap_get(&od->nbs->options, "subnet"); + if (!subnet_str) { + continue; + } + + ovs_be32 subnet, mask; + char *error = ip_parse_masked(subnet_str, &subnet, &mask); + if (error || mask == OVS_BE32_MAX || !ip_is_cidr(mask)) { + static struct vlog_rate_limit rl + = VLOG_RATE_LIMIT_INIT(5, 1); + VLOG_WARN_RL(&rl, "bad 'subnet' %s", subnet_str); + free(error); + continue; + } + + struct ovn_port *op; + for (size_t i = 0; i < od->nbs->n_ports; i++) { + const struct nbrec_logical_switch_port *nbs = + od->nbs->ports[i]; + + if (!nbs) { + continue; + } + + op = ovn_port_find(ports, nbs->name); + if (op && !(op->nbs && op->peer)) { + /* Allocate addresses for logical switch ports that do not + * have a peer. */ + ipam_allocate_addresses(od, op, subnet, mask); + } + } + } + } +} + + +static void join_logical_ports(struct northd_context *ctx, struct hmap *datapaths, struct hmap *ports, struct ovs_list *sb_only, struct ovs_list *nb_only, @@ -597,6 +934,7 @@ join_logical_ports(struct northd_context *ctx, } op->od = od; + ipam_add_port_addresses(od, op); } } else { for (size_t i = 0; i < od->nbr->n_ports; i++) { @@ -682,6 +1020,7 @@ join_logical_ports(struct northd_context *ctx, op->od->router_ports, sizeof *op->od->router_ports * (op->od->n_router_ports + 1)); op->od->router_ports[op->od->n_router_ports++] = op; + ipam_add_port_addresses(op->od, peer); } else if (op->nbr && op->nbr->peer) { op->peer = ovn_port_find(ports, op->nbr->peer); } @@ -2656,6 +2995,7 @@ ovnnb_db_run(struct northd_context *ctx) build_datapaths(ctx, &datapaths); build_ports(ctx, &datapaths, &ports); build_lflows(ctx, &datapaths, &ports); + build_ipam(ctx, &datapaths, &ports); struct ovn_datapath *dp, *next_dp; HMAP_FOR_EACH_SAFE (dp, next_dp, key_node, &datapaths) { @@ -2668,6 +3008,8 @@ ovnnb_db_run(struct northd_context *ctx) ovn_port_destroy(&ports, port); } hmap_destroy(&ports); + + cleanup_macam(&macam); } /* diff --git a/ovn/ovn-nb.ovsschema b/ovn/ovn-nb.ovsschema index 58f04b2..4912b4e 100644 --- a/ovn/ovn-nb.ovsschema +++ b/ovn/ovn-nb.ovsschema @@ -1,7 +1,7 @@ { "name": "OVN_Northbound", - "version": "3.1.0", - "cksum": "1426508118 6135", + "version": "3.2.0", + "cksum": "1510925926 6290", "tables": { "Logical_Switch": { "columns": { @@ -16,6 +16,9 @@ "refType": "strong"}, "min": 0, "max": "unlimited"}}, + "options": { + "type": {"key": "string", "value": "string", + "min": 0, "max": "unlimited"}}, "external_ids": { "type": {"key": "string", "value": "string", "min": 0, "max": "unlimited"}}}, diff --git a/ovn/ovn-nb.xml b/ovn/ovn-nb.xml index 6355c44..9b9bdef 100644 --- a/ovn/ovn-nb.xml +++ b/ovn/ovn-nb.xml @@ -73,6 +73,18 @@ Access control rules that apply to packets within the logical switch. </column> + <group title="Options"> + <p> + Additional options for the logical switch. + </p> + + <column name="options" key="subnet"> + If set, logical ports that are attached to this switch that do not have + a MAC/IPv4 address will automatically be allocated a globally unique + MAC address/unused IPv4 address within the provided subnet. + </column> + </group> + <group title="Common Columns"> <column name="external_ids"> See <em>External IDs</em> at the beginning of this document. @@ -658,7 +670,7 @@ </p> </column> </group> - + <group title="Common Columns"> <column name="external_ids"> See <em>External IDs</em> at the beginning of this document. diff --git a/tests/ovn.at b/tests/ovn.at index 297070c..eda717b 100644 --- a/tests/ovn.at +++ b/tests/ovn.at @@ -3182,3 +3182,159 @@ OVS_APP_EXIT_AND_WAIT([ovs-vswitchd]) OVS_APP_EXIT_AND_WAIT([ovsdb-server]) AT_CLEANUP + +AT_SETUP([ovn -- ipam]) +AT_KEYWORDS([ovnipam]) +AT_SKIP_IF([test $HAVE_PYTHON = no]) +ovn_start + +# Add a port to a switch that does not have a subnet set, then set the +# subnet which should result in an address being allocated for the port. +ovn-nbctl ls-add sw0 +ovn-nbctl lsp-add sw0 p0 +ovn-nbctl add Logical-Switch sw0 options subnet=192.168.1.0/24 +AT_CHECK([ovn-nbctl get Logical-Switch-Port p0 addresses], [0], + [[["0a:00:00:00:00:01 192.168.1.2"]] +]) + +# Add 9 more ports to sw0, addresses should all be unique. +for n in `seq 1 9`; do + ovn-nbctl lsp-add sw0 "p$n" +done +AT_CHECK([ovn-nbctl get Logical-Switch-Port p1 addresses], [0], + [[["0a:00:00:00:00:02 192.168.1.3"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p2 addresses], [0], + [[["0a:00:00:00:00:03 192.168.1.4"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p3 addresses], [0], + [[["0a:00:00:00:00:04 192.168.1.5"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p4 addresses], [0], + [[["0a:00:00:00:00:05 192.168.1.6"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p5 addresses], [0], + [[["0a:00:00:00:00:06 192.168.1.7"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p6 addresses], [0], + [[["0a:00:00:00:00:07 192.168.1.8"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p7 addresses], [0], + [[["0a:00:00:00:00:08 192.168.1.9"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p8 addresses], [0], + [[["0a:00:00:00:00:09 192.168.1.10"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p9 addresses], [0], + [[["0a:00:00:00:00:0a 192.168.1.11"]] +]) + +# Trying similar tests with a second switch. MAC addresses should be unique +# across both switches but IP's only need to be unique within the same switch. +ovn-nbctl ls-add sw1 +ovn-nbctl lsp-add sw1 p10 +ovn-nbctl add Logical-Switch sw1 options subnet=192.168.1.0/24 +AT_CHECK([ovn-nbctl get Logical-Switch-Port p10 addresses], [0], + [[["0a:00:00:00:00:0b 192.168.1.2"]] +]) + +for n in `seq 11 19`; do + ovn-nbctl lsp-add sw1 "p$n" +done +AT_CHECK([ovn-nbctl get Logical-Switch-Port p11 addresses], [0], + [[["0a:00:00:00:00:0c 192.168.1.3"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p12 addresses], [0], + [[["0a:00:00:00:00:0d 192.168.1.4"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p13 addresses], [0], + [[["0a:00:00:00:00:0e 192.168.1.5"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p14 addresses], [0], + [[["0a:00:00:00:00:0f 192.168.1.6"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p15 addresses], [0], + [[["0a:00:00:00:00:10 192.168.1.7"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p16 addresses], [0], + [[["0a:00:00:00:00:11 192.168.1.8"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p17 addresses], [0], + [[["0a:00:00:00:00:12 192.168.1.9"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p18 addresses], [0], + [[["0a:00:00:00:00:13 192.168.1.10"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p19 addresses], [0], + [[["0a:00:00:00:00:14 192.168.1.11"]] +]) + +# Change a port's address to test for the following: address reuse, multiple +# ip's for a single address entry, and addresses set by the user. +ovn-nbctl lsp-set-addresses p0 "0a:00:00:00:00:15 192.168.1.12 192.168.1.14" +ovn-nbctl lsp-add sw0 p20 +ovn-nbctl lsp-add sw0 p21 +AT_CHECK([ovn-nbctl get Logical-Switch-Port p20 addresses], [0], + [[["0a:00:00:00:00:01 192.168.1.2"]] +]) +AT_CHECK([ovn-nbctl get Logical-Switch-Port p21 addresses], [0], + [[["0a:00:00:00:00:16 192.168.1.13"]] +]) + +# Test for logical router port address management. +ovn-nbctl create Logical_Router name=R1 +ovn-nbctl -- --id=@lrp create Logical_Router_port name=sw0 \ +network="192.168.1.1/24" mac=\"0a:00:00:00:00:17\" \ +-- add Logical_Router R1 ports @lrp -- lsp-add sw0 rp-sw0 \ +-- set Logical_Switch_Port rp-sw0 type=router options:router-port=sw0 +ovn-nbctl lsp-add sw0 p22 +AT_CHECK([ovn-nbctl get Logical-Switch-Port p22 addresses], [0], + [[["0a:00:00:00:00:18 192.168.1.15"]] +]) + +# Test for address reuse after logical port is deleted. +ovn-nbctl lsp-del p0 +ovn-nbctl lsp-add sw0 p23 +AT_CHECK([ovn-nbctl get Logical-Switch-Port p23 addresses], [0], + [[["0a:00:00:00:00:15 192.168.1.12"]] +]) + +# Test for only MAC address being assigned. +ovn-nbctl lsp-add sw0 p24 -- lsp-set-addresses p24 "0a:00:00:00:00:19" +AT_CHECK([ovn-nbctl get Logical-Switch-Port p24 addresses], [0], + [[["0a:00:00:00:00:19 192.168.1.14"]] +]) +ovn-nbctl lsp-add sw0 p25 +AT_CHECK([ovn-nbctl get Logical-Switch-Port p25 addresses], [0], + [[["0a:00:00:00:00:1a 192.168.1.16"]] +]) + +# Test for multiple addresses to one logical port. +ovn-nbctl lsp-set-addresses p25 "0a:00:00:00:00:1a 192.168.1.16" \ +"0a:00:00:00:00:1b 192.168.1.17" +ovn-nbctl lsp-add sw0 p26 +AT_CHECK([ovn-nbctl get Logical-Switch-Port p26 addresses], [0], + [[["0a:00:00:00:00:1c 192.168.1.18"]] +]) + +# Test for exhausting subnet address space. +ovn-nbctl ls-add sw2 -- add Logical-Switch sw2 options subnet=172.16.1.0/30 +ovn-nbctl lsp-add sw2 p27 +AT_CHECK([ovn-nbctl get Logical-Switch-Port p27 addresses], [0], + [[["0a:00:00:00:00:1d 172.16.1.2"]] +]) +ovn-nbctl lsp-add sw2 p28 +AT_CHECK([ovn-nbctl get Logical-Switch-Port p28 addresses], [0], + [[[]] +]) + +as ovn-sb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as ovn-nb +OVS_APP_EXIT_AND_WAIT([ovsdb-server]) + +as northd +OVS_APP_EXIT_AND_WAIT([ovn-northd]) + +AT_CLEANUP -- 1.9.1 _______________________________________________ dev mailing list dev@openvswitch.org http://openvswitch.org/mailman/listinfo/dev