ovn-northd processes the list of Port_Bindings and hashes the list of queues per chassis. When it finds a port with qos_parameters and without a queue_id, it allocates a free queue for the chassis that this port belongs. The queue_id information is stored in the options field of Port_binding table. Adds an action set_queue to the ingress table 0 of the logical flows which will be translated to openflow set_queue by ovn-controller
ovn-controller opens the netdev corresponding to the tunnel interface's status:tunnel_egress_iface value and configures a HTB qdisc on it. Then for each SB port_binding that has queue_id set, it allocates a queue with the qos_parameters of that port. It also frees up unused queues. This patch replaces the older approach of policing Signed-off-by: Babu Shanmugam <bscha...@redhat.com> --- ovn/controller/binding.c | 203 +++++++++++++++++++++++++++++++++++++++++------ ovn/lib/actions.c | 29 +++++++ ovn/lib/actions.h | 3 + ovn/northd/ovn-northd.c | 132 ++++++++++++++++++++++++++++-- ovn/ovn-nb.xml | 8 +- ovn/ovn-sb.xml | 31 +++++++- 6 files changed, 368 insertions(+), 38 deletions(-) diff --git a/ovn/controller/binding.c b/ovn/controller/binding.c index 9921a49..97f9aee 100644 --- a/ovn/controller/binding.c +++ b/ovn/controller/binding.c @@ -20,6 +20,7 @@ #include "lib/hmap.h" #include "lib/sset.h" #include "lib/util.h" +#include "lib/netdev.h" #include "lib/vswitch-idl.h" #include "openvswitch/vlog.h" #include "ovn/lib/ovn-sb-idl.h" @@ -27,6 +28,8 @@ VLOG_DEFINE_THIS_MODULE(binding); +#define OVN_QOS_TYPE "linux-htb" + static struct sset all_lports = SSET_INITIALIZER(&all_lports); static bool process_full_binding = false; @@ -37,6 +40,13 @@ binding_reset_processing(void) process_full_binding = true; } +struct qos_queue { + uint32_t queue_id; + uint32_t max_rate; + uint32_t burst; + struct hmap_node node; +}; + void binding_register_ovs_idl(struct ovsdb_idl *ovs_idl) { @@ -50,17 +60,20 @@ binding_register_ovs_idl(struct ovsdb_idl *ovs_idl) ovsdb_idl_add_table(ovs_idl, &ovsrec_table_port); ovsdb_idl_add_column(ovs_idl, &ovsrec_port_col_name); ovsdb_idl_add_column(ovs_idl, &ovsrec_port_col_interfaces); + ovsdb_idl_add_column(ovs_idl, &ovsrec_port_col_qos); ovsdb_idl_add_table(ovs_idl, &ovsrec_table_interface); ovsdb_idl_add_column(ovs_idl, &ovsrec_interface_col_name); ovsdb_idl_add_column(ovs_idl, &ovsrec_interface_col_external_ids); - ovsdb_idl_add_column(ovs_idl, &ovsrec_interface_col_ingress_policing_rate); - ovsdb_idl_add_column(ovs_idl, - &ovsrec_interface_col_ingress_policing_burst); + ovsdb_idl_add_column(ovs_idl, &ovsrec_interface_col_status); + + ovsdb_idl_add_table(ovs_idl, &ovsrec_table_qos); + ovsdb_idl_add_column(ovs_idl, &ovsrec_qos_col_type); } static void -get_local_iface_ids(const struct ovsrec_bridge *br_int, struct shash *lports) +get_local_iface_ids(const struct ovsrec_bridge *br_int, struct shash *lports, + struct sset *egress_ifaces) { int i; @@ -78,13 +91,18 @@ get_local_iface_ids(const struct ovsrec_bridge *br_int, struct shash *lports) iface_rec = port_rec->interfaces[j]; iface_id = smap_get(&iface_rec->external_ids, "iface-id"); - if (!iface_id) { - continue; + if (iface_id) { + shash_add(lports, iface_id, iface_rec); + if (!sset_find(&all_lports, iface_id)) { + sset_add(&all_lports, iface_id); + binding_reset_processing(); + } } - shash_add(lports, iface_id, iface_rec); - if (!sset_find(&all_lports, iface_id)) { - sset_add(&all_lports, iface_id); - binding_reset_processing(); + + /* Check if this is a tunnel interface. */ + if (smap_get(&iface_rec->options, "remote_ip")) { + sset_add(egress_ifaces, + smap_get(&iface_rec->status, "tunnel_egress_iface")); } } } @@ -141,8 +159,8 @@ remove_local_datapath_by_binding(struct hmap *local_datapaths, static void add_local_datapath(struct hmap *local_datapaths, - const struct sbrec_port_binding *binding_rec, - const struct uuid *uuid) + const struct sbrec_port_binding *binding_rec, + const struct uuid *uuid) { if (get_local_datapath(local_datapaths, binding_rec->datapath->tunnel_key)) { @@ -159,21 +177,142 @@ add_local_datapath(struct hmap *local_datapaths, } static void -update_qos(const struct ovsrec_interface *iface_rec, - const struct sbrec_port_binding *pb) +get_qos_params(const struct sbrec_port_binding *pb, struct hmap *queue_map) +{ + uint32_t max_rate = smap_get_int(&pb->options, "qos_max_rate", 0); + uint32_t burst = smap_get_int(&pb->options, "qos_burst", 0); + uint32_t queue_id = smap_get_int(&pb->options, "qdisc_queue_id", 0); + + if ((!max_rate && !burst) || !queue_id) { + /* Qos is not configured for this port. */ + return; + } + + struct qos_queue *node = xzalloc(sizeof *node); + + node->max_rate = max_rate; + node->burst = burst; + node->queue_id = queue_id; + hmap_insert(queue_map, &node->node, hash_int(queue_id, 0)); +} + +static const struct ovsrec_qos * +get_noop_qos(struct controller_ctx *ctx) +{ + const struct ovsrec_qos *qos; + + OVSREC_QOS_FOR_EACH(qos, ctx->ovs_idl) { + if (!strcmp(qos->type, "linux-noop")) { + return qos; + } + } + + qos = ovsrec_qos_insert(ctx->ovs_idl_txn); + ovsrec_qos_set_type(qos, "linux-noop"); + return qos; +} + +static void +set_noop_qos(struct controller_ctx *ctx, struct sset *egress_ifaces) +{ + const struct ovsrec_qos *noop_qos = get_noop_qos(ctx); + const struct ovsrec_port *port; + size_t count = 0; + + OVSREC_PORT_FOR_EACH(port, ctx->ovs_idl) { + if (sset_contains(egress_ifaces, port->name)) { + ovsrec_port_set_qos(port, noop_qos); + count++; + } + if (sset_count(egress_ifaces) == count) { + break; + } + } +} + +static void +setup_qos(const char *egress_iface, struct hmap *queue_map) { - int rate = smap_get_int(&pb->options, "policing_rate", 0); - int burst = smap_get_int(&pb->options, "policing_burst", 0); + struct netdev *netdev_phy; + + if (!egress_iface) { + /* Queues cannot be configured. */ + return; + } + VLOG_WARN("Setting up qos on %s", egress_iface); + + if (netdev_open(egress_iface, NULL, &netdev_phy) != 0) { + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1); + VLOG_WARN_RL(&rl, "Unable to open netdev %s\n", egress_iface); + return; + } + + /* Check and configure qdisc. */ + const char *qdisc_type; + struct smap qdisc_details; + + smap_init(&qdisc_details); + if (netdev_get_qos(netdev_phy, &qdisc_type, &qdisc_details) != 0 || + qdisc_type[0] == '\0') { + /* Qos is not supported. */ + return; + } + if (strcmp(qdisc_type, OVN_QOS_TYPE)) { + netdev_set_qos(netdev_phy, OVN_QOS_TYPE, &qdisc_details); + } + + /* Check and delete if needed. */ + struct netdev_queue_dump dump; + unsigned int queue_id; + struct smap queue_details; + struct qos_queue *sb_info; + struct hmap consistent_queues; + + smap_init(&queue_details); + hmap_init(&consistent_queues); + NETDEV_QUEUE_FOR_EACH(&queue_id, &queue_details, &dump, netdev_phy) { + bool is_queue_needed = false; + + HMAP_FOR_EACH_WITH_HASH (sb_info, node, hash_int(queue_id, 0), + queue_map) { + is_queue_needed = true; + if (sb_info->max_rate == + smap_get_int(&queue_details, "max-rate", 0) + && sb_info->burst == smap_get_int(&queue_details, "burst", 0)) { + /* This queue is consistent. */ + hmap_insert(&consistent_queues, &sb_info->node, + hash_int(queue_id, 0)); + break; + } + } - ovsrec_interface_set_ingress_policing_rate(iface_rec, MAX(0, rate)); - ovsrec_interface_set_ingress_policing_burst(iface_rec, MAX(0, burst)); + if (!is_queue_needed) { + netdev_delete_queue(netdev_phy, queue_id); + } + } + + /* Create/Update queues. */ + HMAP_FOR_EACH(sb_info, node, queue_map) { + if (hmap_contains(&consistent_queues, &sb_info->node)) { + hmap_remove(&consistent_queues, &sb_info->node); + continue; + } + + smap_clear(&queue_details); + smap_add_format(&queue_details, "max-rate", "%d", sb_info->max_rate); + smap_add_format(&queue_details, "burst", "%d", sb_info->burst); + netdev_set_queue(netdev_phy, sb_info->queue_id, &queue_details); + } + smap_destroy(&queue_details); + hmap_destroy(&consistent_queues); + netdev_close(netdev_phy); } static void consider_local_datapath(struct controller_ctx *ctx, struct shash *lports, const struct sbrec_chassis *chassis_rec, const struct sbrec_port_binding *binding_rec, - struct hmap *local_datapaths) + struct hmap *qos_map, struct hmap *local_datapaths) { const struct ovsrec_interface *iface_rec = shash_find_and_delete(lports, binding_rec->logical_port); @@ -186,8 +325,8 @@ consider_local_datapath(struct controller_ctx *ctx, struct shash *lports, } add_local_datapath(local_datapaths, binding_rec, &binding_rec->header_.uuid); - if (iface_rec && ctx->ovs_idl_txn) { - update_qos(iface_rec, binding_rec); + if (iface_rec && qos_map && ctx->ovs_idl_txn) { + get_qos_params(binding_rec, qos_map); } if (binding_rec->chassis == chassis_rec) { return; @@ -227,15 +366,18 @@ binding_run(struct controller_ctx *ctx, const struct ovsrec_bridge *br_int, { const struct sbrec_chassis *chassis_rec; const struct sbrec_port_binding *binding_rec; + struct sset egress_ifaces = SSET_INITIALIZER(&egress_ifaces); + struct hmap qos_map; chassis_rec = get_chassis(ctx->ovnsb_idl, chassis_id); if (!chassis_rec) { return; } + hmap_init(&qos_map); struct shash lports = SHASH_INITIALIZER(&lports); if (br_int) { - get_local_iface_ids(br_int, &lports); + get_local_iface_ids(br_int, &lports, &egress_ifaces); } else { /* We have no integration bridge, therefore no local logical ports. * We'll remove our chassis from all port binding records below. */ @@ -249,7 +391,8 @@ binding_run(struct controller_ctx *ctx, const struct ovsrec_bridge *br_int, HMAP_INITIALIZER(&keep_local_datapath_by_uuid); SBREC_PORT_BINDING_FOR_EACH(binding_rec, ctx->ovnsb_idl) { consider_local_datapath(ctx, &lports, chassis_rec, binding_rec, - local_datapaths); + sset_is_empty(&egress_ifaces) ? NULL : + &qos_map, local_datapaths); struct local_datapath *ld = xzalloc(sizeof *ld); ld->uuid = &binding_rec->header_.uuid; hmap_insert(&keep_local_datapath_by_uuid, &ld->uuid_hmap_node, @@ -270,11 +413,23 @@ binding_run(struct controller_ctx *ctx, const struct ovsrec_bridge *br_int, remove_local_datapath_by_binding(local_datapaths, binding_rec); } else { consider_local_datapath(ctx, &lports, chassis_rec, binding_rec, - local_datapaths); + sset_is_empty(&egress_ifaces) ? NULL : + &qos_map, local_datapaths); } } } + if (!sset_is_empty(&egress_ifaces)) { + const char *entry; + + set_noop_qos(ctx, &egress_ifaces); + SSET_FOR_EACH(entry, &egress_ifaces) { + setup_qos(entry, &qos_map); + } + } + + sset_destroy(&egress_ifaces); + hmap_destroy(&qos_map); shash_destroy(&lports); } diff --git a/ovn/lib/actions.c b/ovn/lib/actions.c index 569970e..bb64391 100644 --- a/ovn/lib/actions.c +++ b/ovn/lib/actions.c @@ -624,6 +624,33 @@ parse_put_dhcp_opts_action(struct action_context *ctx, } static void +parse_set_queue_action(struct action_context *ctx) +{ + int queue_id; + + if (!lexer_match(ctx->lexer, LEX_T_LPAREN)) { + action_syntax_error(ctx, "expecting `('"); + return; + } + if (!action_get_int(ctx, &queue_id)) { + return; + } + if (!lexer_match(ctx->lexer, LEX_T_RPAREN)) { + action_syntax_error(ctx, "expecting `)'"); + return; + } + if (queue_id < QDISC_MIN_QUEUE_ID || queue_id > QDISC_MAX_QUEUE_ID) { + action_error(ctx, "Queue ID %d for set_queue is " + "not in valid range %d to %d.", + queue_id, QDISC_MIN_QUEUE_ID, QDISC_MAX_QUEUE_ID); + return; + } + + struct ofpact_queue *set_queue = ofpact_put_SET_QUEUE(ctx->ofpacts); + set_queue->queue_id = queue_id; +} + +static void emit_ct(struct action_context *ctx, bool recirc_next, bool commit) { struct ofpact_conntrack *ct = ofpact_put_CT(ctx->ofpacts); @@ -768,6 +795,8 @@ parse_action(struct action_context *ctx) parse_get_arp_action(ctx); } else if (lexer_match_id(ctx->lexer, "put_arp")) { parse_put_arp_action(ctx); + } else if (lexer_match_id(ctx->lexer, "set_queue")) { + parse_set_queue_action(ctx); } else { action_syntax_error(ctx, "expecting action"); } diff --git a/ovn/lib/actions.h b/ovn/lib/actions.h index f49e15e..14c23e8 100644 --- a/ovn/lib/actions.h +++ b/ovn/lib/actions.h @@ -22,6 +22,9 @@ #include "compiler.h" #include "util.h" +#define QDISC_MIN_QUEUE_ID (1) +#define QDISC_MAX_QUEUE_ID (0xf000) + struct expr; struct lexer; struct ofpbuf; diff --git a/ovn/northd/ovn-northd.c b/ovn/northd/ovn-northd.c index c2cf15e..9da8614 100644 --- a/ovn/northd/ovn-northd.c +++ b/ovn/northd/ovn-northd.c @@ -30,6 +30,7 @@ #include "ovn/lib/ovn-nb-idl.h" #include "ovn/lib/ovn-sb-idl.h" #include "ovn/lib/ovn-util.h" +#include "ovn/lib/actions.h" #include "packets.h" #include "poll-loop.h" #include "smap.h" @@ -237,6 +238,78 @@ allocate_tnlid(struct hmap *set, const char *name, uint32_t max, return 0; } +struct ovn_chassis_qdisc_queues { + uint32_t queue_id; + struct hmap_node key_node; +}; + +static void +destroy_chassis_queues(struct hmap *set) +{ + struct ovn_chassis_qdisc_queues *node; + HMAP_FOR_EACH_POP (node, key_node, set) { + free(node); + } + hmap_destroy(set); +} + +static void +add_chassis_queue(struct hmap *set, const char *chassis_name, + uint32_t queue_id) +{ + struct ovn_chassis_qdisc_queues *node = xmalloc(sizeof *node); + node->queue_id = queue_id; + hmap_insert(set, &node->key_node, hash_string(chassis_name, 0)); +} + +static bool +chassis_queueid_in_use(const struct hmap *set, const char *chassis, + uint32_t queue_id) +{ + const struct ovn_chassis_qdisc_queues *node; + HMAP_FOR_EACH_WITH_HASH (node, key_node, hash_string(chassis, 0), set) { + if (node->queue_id == queue_id) { + return true; + } + } + return false; +} + +static uint32_t +allocate_chassis_queueid(struct hmap *set, const char *chassis) +{ + for (uint32_t queue_id = QDISC_MIN_QUEUE_ID; + queue_id <= QDISC_MAX_QUEUE_ID; + queue_id++) { + if (!chassis_queueid_in_use(set, chassis, queue_id)) { + add_chassis_queue(set, chassis, queue_id); + return queue_id; + } + } + + static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1); + VLOG_WARN_RL(&rl, "all %s queue ids exhausted", chassis); + return 0; +} + +static void +free_chassis_queueid(struct hmap *set, const char * chassis, uint32_t queue_id) +{ + struct ovn_chassis_qdisc_queues *node; + HMAP_FOR_EACH_WITH_HASH (node, key_node, hash_string(chassis, 0), set) { + if (node->queue_id == queue_id) { + hmap_remove(set, &node->key_node); + break; + } + } +} + +static inline bool +port_has_qos_params(struct smap * opts) { + return (smap_get(opts, "qos_max_rate") || + smap_get(opts, "qos_burst")); +} + /* The 'key' comes from nbs->header_.uuid or nbr->header_.uuid or * sb->external_ids:logical-switch. */ struct ovn_datapath { @@ -559,6 +632,7 @@ ovn_port_allocate_key(struct ovn_datapath *od) static void join_logical_ports(struct northd_context *ctx, struct hmap *datapaths, struct hmap *ports, + struct hmap *chassis_qdisc_queues, struct ovs_list *sb_only, struct ovs_list *nb_only, struct ovs_list *both) { @@ -590,6 +664,15 @@ join_logical_ports(struct northd_context *ctx, } op->nbs = nbs; ovs_list_remove(&op->list); + + uint32_t queue_id = smap_get_int(&op->sb->options, + "qdisc_queue_id", 0); + if (queue_id && op->sb->chassis) { + add_chassis_queue( + chassis_qdisc_queues, op->sb->chassis->name, + queue_id); + } + ovs_list_push_back(both, &op->list); } else { op = ovn_port_create(ports, nbs->name, nbs, NULL, NULL); @@ -689,7 +772,8 @@ join_logical_ports(struct northd_context *ctx, } static void -ovn_port_update_sbrec(const struct ovn_port *op) +ovn_port_update_sbrec(const struct ovn_port *op, + struct hmap *chassis_qdisc_queues) { sbrec_port_binding_set_datapath(op->sb, op->od->sb); if (op->nbr) { @@ -717,8 +801,31 @@ ovn_port_update_sbrec(const struct ovn_port *op) sbrec_port_binding_set_mac(op->sb, NULL, 0); } else { if (strcmp(op->nbs->type, "router")) { + uint32_t queue_id = smap_get_int( + &op->sb->options, "qdisc_queue_id", 0); + struct smap options; + + smap_clone(&options, &op->nbs->options); + if (op->sb->chassis && port_has_qos_params(&options) + && !queue_id) { + queue_id = allocate_chassis_queueid(chassis_qdisc_queues, + op->sb->chassis->name); + } + if (!port_has_qos_params(&options) && queue_id) { + /* Free this queue. */ + free_chassis_queueid(chassis_qdisc_queues, + op->sb->chassis->name, + queue_id); + queue_id = 0; + } + + if (queue_id) { + /* Only when there is a valid queue. */ + smap_add_format(&options, + "qdisc_queue_id", "%d", queue_id); + } sbrec_port_binding_set_type(op->sb, op->nbs->type); - sbrec_port_binding_set_options(op->sb, &op->nbs->options); + sbrec_port_binding_set_options(op->sb, &options); } else { const char *chassis = NULL; if (op->peer && op->peer->od && op->peer->od->nbr) { @@ -765,14 +872,18 @@ build_ports(struct northd_context *ctx, struct hmap *datapaths, struct hmap *ports) { struct ovs_list sb_only, nb_only, both; + struct hmap chassis_qdisc_queues; + + hmap_init(&chassis_qdisc_queues); - join_logical_ports(ctx, datapaths, ports, &sb_only, &nb_only, &both); + join_logical_ports(ctx, datapaths, ports, &chassis_qdisc_queues, + &sb_only, &nb_only, &both); /* For logical ports that are in both databases, update the southbound * record based on northbound data. Also index the in-use tunnel_keys. */ struct ovn_port *op, *next; LIST_FOR_EACH_SAFE (op, next, list, &both) { - ovn_port_update_sbrec(op); + ovn_port_update_sbrec(op, &chassis_qdisc_queues); add_tnlid(&op->od->port_tnlids, op->sb->tunnel_key); if (op->sb->tunnel_key > op->od->port_key_hint) { @@ -788,7 +899,7 @@ build_ports(struct northd_context *ctx, struct hmap *datapaths, } op->sb = sbrec_port_binding_insert(ctx->ovnsb_txn); - ovn_port_update_sbrec(op); + ovn_port_update_sbrec(op, &chassis_qdisc_queues); sbrec_port_binding_set_logical_port(op->sb, op->key); sbrec_port_binding_set_tunnel_key(op->sb, tunnel_key); @@ -800,6 +911,8 @@ build_ports(struct northd_context *ctx, struct hmap *datapaths, sbrec_port_binding_delete(op->sb); ovn_port_destroy(ports, op); } + + destroy_chassis_queues(&chassis_qdisc_queues); } #define OVN_MIN_MULTICAST 32768 @@ -1535,12 +1648,19 @@ build_lswitch_flows(struct hmap *datapaths, struct hmap *ports, } struct ds match = DS_EMPTY_INITIALIZER; + struct ds action = DS_EMPTY_INITIALIZER; + ds_put_format(&match, "inport == %s", op->json_key); build_port_security_l2( "eth.src", op->nbs->port_security, op->nbs->n_port_security, &match); + const char *queue_id = smap_get(&op->sb->options, "qdisc_queue_id"); + if (queue_id) { + ds_put_format(&action, "set_queue(%s);", queue_id); + } + ds_put_cstr(&action, "next;"); ovn_lflow_add(lflows, op->od, S_SWITCH_IN_PORT_SEC_L2, 50, - ds_cstr(&match), "next;"); + ds_cstr(&match), ds_cstr(&action)); ds_destroy(&match); if (op->nbs->n_port_security) { diff --git a/ovn/ovn-nb.xml b/ovn/ovn-nb.xml index 6355c44..308ae88 100644 --- a/ovn/ovn-nb.xml +++ b/ovn/ovn-nb.xml @@ -195,14 +195,14 @@ (empty string) </p> - <column name="options" key="policing_rate"> + <column name="options" key="qos_max_rate"> If set, indicates the maximum rate for data sent from this interface, - in kbps. Data exceeding this rate is dropped. + in bit/s. The traffic will be shaped according to this limit. </column> - <column name="options" key="policing_burst"> + <column name="options" key="qos_burst"> If set, indicates the maximum burst size for data sent from this - interface, in kb. + interface, in bits. </column> </group> </group> diff --git a/ovn/ovn-sb.xml b/ovn/ovn-sb.xml index f330374..6655fab 100644 --- a/ovn/ovn-sb.xml +++ b/ovn/ovn-sb.xml @@ -1065,6 +1065,23 @@ </dd> <dt> + <code>set_queue(<var>queue_number</var>);</code> + </dt> + + <dd> + <p> + <b>Parameters</b>: Queue number <var>queue_number</var>, 32-bit + </p> + + <p> + This is equivalent to Openflow set_queue. queue_number should be + in the range of 1 to 61440 + </p> + + <p><b>Example:</b> <code>set_queue(10);</code></p> + </dd> + + <dt> <code><var>R</var> = put_dhcp_opts(<code>offerip</code> = <var>IP</var>, <var>D1</var> = <var>V1</var>, <var>D2</var> = <var>V2</var>, ..., <var>Dn</var> = <var>Vn</var>);</code> </dt> @@ -1511,14 +1528,20 @@ tcp.flags = RST; (empty string) </p> - <column name="options" key="policing_rate"> + <column name="options" key="qos_max_rate"> If set, indicates the maximum rate for data sent from this interface, - in kbps. Data exceeding this rate is dropped. + in bit/s. The traffic will be shaped according to this limit. </column> - <column name="options" key="policing_burst"> + <column name="options" key="qos_burst"> If set, indicates the maximum burst size for data sent from this - interface, in kb. + interface, in bits. + </column> + + <column name="options" key="qdisc_queue_id"> + Indicates the queue number on the physical device. This is same as the + queue_id used in OpenFlow in struct ofp_action_enqueue. Value should + be in the range of 1 to 61440. </column> </group> -- 2.5.5 _______________________________________________ dev mailing list dev@openvswitch.org http://openvswitch.org/mailman/listinfo/dev