Il 11/03/2013 05:33, michael.r.hines.mrhi...@linux.vnet.ibm.com ha scritto: > From: "Michael R. Hines" <mrhi...@us.ibm.com> > > Use 'migrate rdma:host:port' to begin the migration. > > The TCP control channel has finally been eliminated > when RDMA is used. Documentation of the use of SEND message > for transferring device state is covered in docs/rdma.txt > > Signed-off-by: Michael R. Hines <mrhi...@us.ibm.com> > --- > migration-rdma.c | 198 > ++++++++++++++++++++++++++++++++++++++++++++++++++++++ > 1 file changed, 198 insertions(+) > create mode 100644 migration-rdma.c > > diff --git a/migration-rdma.c b/migration-rdma.c > new file mode 100644 > index 0000000..822b17a > --- /dev/null > +++ b/migration-rdma.c > @@ -0,0 +1,198 @@ > +/* > + * Copyright (C) 2013 Michael R. Hines <mrhi...@us.ibm.com> > + * Copyright (C) 2013 Jiuxing Liu <j...@us.ibm.com> > + * > + * This program is free software; you can redistribute it and/or modify > + * it under the terms of the GNU General Public License as published by > + * the Free Software Foundation; under version 2 of the License. > + * > + * This program is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU General Public License for more details. > + * > + * You should have received a copy of the GNU General Public License > + * along with this program; if not, see <http://www.gnu.org/licenses/>. > + */ > +#include "migration/rdma.h" > +#include "qemu-common.h" > +#include "migration/migration.h" > +#include "migration/qemu-file.h" > +#include <stdio.h> > +#include <sys/types.h> > +#include <sys/socket.h> > +#include <netdb.h> > +#include <arpa/inet.h> > +#include <string.h> > + > +//#define DEBUG_MIGRATION_RDMA > + > +#ifdef DEBUG_MIGRATION_RDMA > +#define DPRINTF(fmt, ...) \ > + do { printf("migration-rdma: " fmt, ## __VA_ARGS__); } while (0) > +#else > +#define DPRINTF(fmt, ...) \ > + do { } while (0) > +#endif > + > +static int rdma_accept_incoming_migration(RDMAData *rdma, Error **errp) > +{ > + int ret; > + > + ret = qemu_rdma_migrate_listen(rdma, rdma->host, rdma->port); > + if (ret) { > + qemu_rdma_print("rdma migration: error listening!"); > + goto err_rdma_server_wait; > + } > + > + ret = qemu_rdma_alloc_qp(&rdma->rdma_ctx); > + if (ret) { > + qemu_rdma_print("rdma migration: error allocating qp!"); > + goto err_rdma_server_wait; > + } > + > + ret = qemu_rdma_migrate_accept(&rdma->rdma_ctx, NULL, NULL, NULL, 0); > + if (ret) { > + qemu_rdma_print("rdma migration: error accepting connection!"); > + goto err_rdma_server_wait; > + } > + > + ret = qemu_rdma_post_recv_qemu_file(rdma); > + if (ret) { > + qemu_rdma_print("rdma migration: error posting second qemu file > recv!"); > + goto err_rdma_server_wait; > + } > + > + ret = qemu_rdma_post_send_remote_info(rdma); > + if (ret) { > + qemu_rdma_print("rdma migration: error sending remote info!"); > + goto err_rdma_server_wait; > + } > + > + ret = qemu_rdma_wait_for_wrid(rdma, RDMA_WRID_SEND_REMOTE_INFO); > + if (ret < 0) { > + qemu_rdma_print("rdma migration: polling remote info error!"); > + goto err_rdma_server_wait; > + } > + > + rdma->total_bytes = 0; > + rdma->enabled = 1; > + qemu_rdma_dump_gid("server_connect", rdma->rdma_ctx.cm_id); > + return 0; > + > +err_rdma_server_wait: > + qemu_rdma_cleanup(rdma); > + return -1; > + > +} > + > +int rdma_start_incoming_migration(const char * host_port, Error **errp) > +{ > + QEMUFile *f; > + int ret; > + RDMAData rdma; > + > + if ((ret = qemu_rdma_data_init(&rdma, host_port, errp)) < 0) > + return ret; > + > + ret = qemu_rdma_server_init(&rdma, NULL); > + > + DPRINTF("Starting RDMA-based incoming migration\n"); > + > + if (!ret) { > + DPRINTF("qemu_rdma_server_init success\n"); > + ret = qemu_rdma_server_prepare(&rdma, NULL); > + > + if (!ret) { > + DPRINTF("qemu_rdma_server_prepare success\n"); > + > + ret = rdma_accept_incoming_migration(&rdma, NULL); > + if(!ret) > + DPRINTF("qemu_rdma_accept_incoming_migration success\n"); > + f = qemu_fopen_rdma(&rdma); > + if (f == NULL) { > + fprintf(stderr, "could not qemu_fopen RDMA\n"); > + ret = -EIO; > + } > + > + process_incoming_migration(f); > + } > + } > + > + return ret; > +} > + > +/* > + * Not sure what this is for yet... > + */ > +static int qemu_rdma_errno(MigrationState *s) > +{ > + return 0; > +} > + > +/* > + * SEND messages for device state only. > + * pc.ram is handled elsewhere... > + */ > +static int qemu_rdma_send(MigrationState *s, const void * buf, size_t size) > +{ > + size_t len, remaining = size; > + uint8_t * data = (void *) buf; > + > + if (qemu_rdma_write_flush(&s->rdma) < 0) { > + qemu_file_set_error(s->file, -EIO); > + return -EIO; > + } > + > + while(remaining) { > + len = MIN(remaining, RDMA_SEND_INCREMENT); > + remaining -= len; > + DPRINTF("iter Sending %" PRId64 "-byte SEND\n", len); > + > + if(qemu_rdma_exchange_send(&s->rdma, data, len) < 0) > + return -EINVAL; > + > + data += len; > + } > + > + return size; > +} > + > +static int qemu_rdma_close(MigrationState *s) > +{ > + DPRINTF("qemu_rdma_close\n"); > + qemu_rdma_cleanup(&s->rdma); > + return 0; > +} > + > +void rdma_start_outgoing_migration(MigrationState *s, const char *host_port, > Error **errp) > +{ > + int ret; > + > +#ifndef CONFIG_RDMA > + error_set(errp, QERR_FEATURE_DISABLED, "rdma migration"); > + return; > +#endif > + > + if (qemu_rdma_data_init(&s->rdma, host_port, errp) < 0) > + return; > + > + ret = qemu_rdma_client_init(&s->rdma, NULL); > + if(!ret) { > + DPRINTF("qemu_rdma_client_init success\n"); > + ret = qemu_rdma_client_connect(&s->rdma, NULL); > + > + if(!ret) { > + s->get_error = qemu_rdma_errno; > + s->write = qemu_rdma_send; > + s->close = qemu_rdma_close; > + s->fd = -2; > + DPRINTF("qemu_rdma_client_connect success\n"); > + migrate_fd_connect(s); > + return; > + } > + } > + > + s->fd = -1; > + migrate_fd_error(s); > +} >
Note that as soon as the pending migration patches hit upstream, almost all of this code will move to QEMUFileRDMA (in particular qemu_rdma_send and qemu_rdma_close). It should become simpler. Paolo
