Re: [QEMU][PATCH v1 5/7] memory: add MemoryRegion map and unmap callbacks

[Juergen Gross]

On 10.10.23 02:17, Stefano Stabellini wrote:

On Thu, 5 Oct 2023, Vikram Garhwal wrote:

From: Juergen Gross 

In order to support mapping and unmapping guest memory dynamically to
and from qemu during address_space_[un]map() operations add the map()
and unmap() callbacks to MemoryRegionOps.

Those will be used e.g. for Xen grant mappings when performing guest
I/Os.

Signed-off-by: Juergen Gross 
Signed-off-by: Vikram Garhwal 


Can't we just use the existing Xen hooks in qemu_ram_ptr_length and
xen_invalidate_map_cache_entry? Do we really need new ones?


I tried your idea first and it didn't work out.

The existing hooks are invoked not only when explicitly [un]mapping memory
regions, but in some other cases, too. Have a look for qemu_ram_ptr_length()
call in flatview_write_continue().


Juergen


OpenPGP_0xB0DE9DD628BF132F.asc
Description: OpenPGP public key


OpenPGP_signature.asc
Description: OpenPGP digital signature

[PATCH v9 01/23] target/riscv: Move MISA limits to class

[Akihiko Odaki]

MISA limits are common for all instances of a RISC-V CPU class so they
are better put into class.

Signed-off-by: Akihiko Odaki 
---
 target/riscv/cpu-qom.h   |   2 +
 target/riscv/cpu.h   |   2 -
 hw/riscv/boot.c  |   2 +-
 target/riscv/cpu.c   | 212 +++
 target/riscv/csr.c   |   3 +-
 target/riscv/gdbstub.c   |  12 ++-
 target/riscv/machine.c   |  11 +-
 target/riscv/translate.c |   3 +-
 8 files changed, 167 insertions(+), 80 deletions(-)

diff --git a/target/riscv/cpu-qom.h b/target/riscv/cpu-qom.h
index 04af50983e..266a07f5be 100644
--- a/target/riscv/cpu-qom.h
+++ b/target/riscv/cpu-qom.h
@@ -67,5 +67,7 @@ struct RISCVCPUClass {
 /*< public >*/
 DeviceRealize parent_realize;
 ResettablePhases parent_phases;
+uint32_t misa_mxl_max;  /* max mxl for this cpu */
+uint32_t misa_ext_mask; /* max ext for this cpu */
 };
 #endif /* RISCV_CPU_QOM_H */
diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h
index ef9cf21c0c..9f9cb6cd2a 100644
--- a/target/riscv/cpu.h
+++ b/target/riscv/cpu.h
@@ -155,9 +155,7 @@ struct CPUArchState {
 
 /* RISCVMXL, but uint32_t for vmstate migration */
 uint32_t misa_mxl;  /* current mxl */
-uint32_t misa_mxl_max;  /* max mxl for this cpu */
 uint32_t misa_ext;  /* current extensions */
-uint32_t misa_ext_mask; /* max ext for this cpu */
 uint32_t xl;/* current xlen */
 
 /* 128-bit helpers upper part return value */
diff --git a/hw/riscv/boot.c b/hw/riscv/boot.c
index 52bf8e67de..b7cf08f479 100644
--- a/hw/riscv/boot.c
+++ b/hw/riscv/boot.c
@@ -36,7 +36,7 @@
 
 bool riscv_is_32bit(RISCVHartArrayState *harts)
 {
-return harts->harts[0].env.misa_mxl_max == MXL_RV32;
+return RISCV_CPU_GET_CLASS(&harts->harts[0])->misa_mxl_max == MXL_RV32;
 }
 
 /*
diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c
index f5572704de..3bb1ce90f9 100644
--- a/target/riscv/cpu.c
+++ b/target/riscv/cpu.c
@@ -41,6 +41,11 @@
 /* RISC-V CPU definitions */
 static const char riscv_single_letter_exts[] = "IEMAFDQCPVH";
 
+typedef struct RISCVCPUClassData {
+RISCVMXL misa_mxl_max;
+uint32_t misa_ext_mask;
+} RISCVCPUClassData;
+
 struct isa_ext_data {
 const char *name;
 int min_version;
@@ -271,12 +276,6 @@ const char *riscv_cpu_get_trap_name(target_ulong cause, 
bool async)
 }
 }
 
-static void set_misa(CPURISCVState *env, RISCVMXL mxl, uint32_t ext)
-{
-env->misa_mxl_max = env->misa_mxl = mxl;
-env->misa_ext_mask = env->misa_ext = ext;
-}
-
 #ifndef CONFIG_USER_ONLY
 static uint8_t satp_mode_from_str(const char *satp_mode_str)
 {
@@ -371,15 +370,20 @@ static void set_satp_mode_default_map(RISCVCPU *cpu)
 }
 #endif
 
+static const RISCVCPUClassData riscv_any_cpu_class_data = {
+#if defined(TARGET_RISCV32)
+.misa_mxl_max = MXL_RV32,
+.misa_ext_mask = RVI | RVM | RVA | RVF | RVD | RVC | RVU
+#else
+.misa_mxl_max = MXL_RV64,
+.misa_ext_mask = RVI | RVM | RVA | RVF | RVD | RVC | RVU
+#endif
+};
+
 static void riscv_any_cpu_init(Object *obj)
 {
 RISCVCPU *cpu = RISCV_CPU(obj);
 CPURISCVState *env = &cpu->env;
-#if defined(TARGET_RISCV32)
-set_misa(env, MXL_RV32, RVI | RVM | RVA | RVF | RVD | RVC | RVU);
-#elif defined(TARGET_RISCV64)
-set_misa(env, MXL_RV64, RVI | RVM | RVA | RVF | RVD | RVC | RVU);
-#endif
 
 #ifndef CONFIG_USER_ONLY
 set_satp_mode_max_supported(RISCV_CPU(obj),
@@ -397,11 +401,13 @@ static void riscv_any_cpu_init(Object *obj)
 }
 
 #if defined(TARGET_RISCV64)
+static const RISCVCPUClassData rv64_base_cpu_class_data = {
+.misa_mxl_max = MXL_RV64
+};
+
 static void rv64_base_cpu_init(Object *obj)
 {
 CPURISCVState *env = &RISCV_CPU(obj)->env;
-/* We set this in the realise function */
-set_misa(env, MXL_RV64, 0);
 riscv_cpu_add_user_properties(obj);
 /* Set latest version of privileged specification */
 env->priv_ver = PRIV_VERSION_LATEST;
@@ -410,11 +416,15 @@ static void rv64_base_cpu_init(Object *obj)
 #endif
 }
 
+static const RISCVCPUClassData rv64_sifive_u_cpu_class_data = {
+.misa_mxl_max = MXL_RV64,
+.misa_ext_mask = RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU
+};
+
 static void rv64_sifive_u_cpu_init(Object *obj)
 {
 RISCVCPU *cpu = RISCV_CPU(obj);
 CPURISCVState *env = &cpu->env;
-set_misa(env, MXL_RV64, RVI | RVM | RVA | RVF | RVD | RVC | RVS | RVU);
 env->priv_ver = PRIV_VERSION_1_10_0;
 #ifndef CONFIG_USER_ONLY
 set_satp_mode_max_supported(RISCV_CPU(obj), VM_1_10_SV39);
@@ -427,12 +437,16 @@ static void rv64_sifive_u_cpu_init(Object *obj)
 cpu->cfg.pmp = true;
 }
 
+static const RISCVCPUClassData rv64_sifive_e_cpu_class_data = {
+.misa_mxl_max = MXL_RV64,
+.misa_ext_mask = RVI | RVM | RVA | RVC | RVU
+};
+
 static void rv64_sifive_e_cpu_init(Object *obj)
 {
 CPURISCVState *env = &RISCV_CPU(obj)->env;
 RISCVCPU *cpu = RISCV_CPU(obj);
 
-set_misa(env, MXL_RV64, RVI | RVM | RVA | RVC | RVU);
 env->priv_ver = 

[PATCH v9 07/23] target/arm: Use GDBFeature for dynamic XML

[Akihiko Odaki]

In preparation for a change to use GDBFeature as a parameter of
gdb_register_coprocessor(), convert the internal representation of
dynamic feature from plain XML to GDBFeature.

Signed-off-by: Akihiko Odaki 
Acked-by: Richard Henderson 
---
 target/arm/cpu.h   |  21 +++
 target/arm/internals.h |   2 +-
 target/arm/gdbstub.c   | 134 ++---
 target/arm/gdbstub64.c |  90 ---
 4 files changed, 109 insertions(+), 138 deletions(-)

diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index a9edfb8353..3702ddaab8 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -25,6 +25,7 @@
 #include "hw/registerfields.h"
 #include "cpu-qom.h"
 #include "exec/cpu-defs.h"
+#include "exec/gdbstub.h"
 #include "qapi/qapi-types-common.h"
 
 /* ARM processors have a weak memory model */
@@ -136,23 +137,21 @@ enum {
  */
 
 /**
- * DynamicGDBXMLInfo:
- * @desc: Contains the XML descriptions.
- * @num: Number of the registers in this XML seen by GDB.
+ * DynamicGDBFeatureInfo:
+ * @desc: Contains the feature descriptions.
  * @data: A union with data specific to the set of registers
  *@cpregs_keys: Array that contains the corresponding Key of
  *  a given cpreg with the same order of the cpreg
  *  in the XML description.
  */
-typedef struct DynamicGDBXMLInfo {
-char *desc;
-int num;
+typedef struct DynamicGDBFeatureInfo {
+GDBFeature desc;
 union {
 struct {
 uint32_t *keys;
 } cpregs;
 } data;
-} DynamicGDBXMLInfo;
+} DynamicGDBFeatureInfo;
 
 /* CPU state for each instance of a generic timer (in cp15 c14) */
 typedef struct ARMGenericTimer {
@@ -880,10 +879,10 @@ struct ArchCPU {
 uint64_t *cpreg_vmstate_values;
 int32_t cpreg_vmstate_array_len;
 
-DynamicGDBXMLInfo dyn_sysreg_xml;
-DynamicGDBXMLInfo dyn_svereg_xml;
-DynamicGDBXMLInfo dyn_m_systemreg_xml;
-DynamicGDBXMLInfo dyn_m_secextreg_xml;
+DynamicGDBFeatureInfo dyn_sysreg_feature;
+DynamicGDBFeatureInfo dyn_svereg_feature;
+DynamicGDBFeatureInfo dyn_m_systemreg_feature;
+DynamicGDBFeatureInfo dyn_m_secextreg_feature;
 
 /* Timers used by the generic (architected) timer */
 QEMUTimer *gt_timer[NUM_GTIMERS];
diff --git a/target/arm/internals.h b/target/arm/internals.h
index 1dd9182a54..09350e96da 100644
--- a/target/arm/internals.h
+++ b/target/arm/internals.h
@@ -1440,7 +1440,7 @@ static inline uint64_t pmu_counter_mask(CPUARMState *env)
 }
 
 #ifdef TARGET_AARCH64
-int arm_gen_dynamic_svereg_xml(CPUState *cpu, int base_reg);
+GDBFeature *arm_gen_dynamic_svereg_feature(CPUState *cpu);
 int aarch64_gdb_get_sve_reg(CPUARMState *env, GByteArray *buf, int reg);
 int aarch64_gdb_set_sve_reg(CPUARMState *env, uint8_t *buf, int reg);
 int aarch64_gdb_get_fpu_reg(CPUARMState *env, GByteArray *buf, int reg);
diff --git a/target/arm/gdbstub.c b/target/arm/gdbstub.c
index b7ace24bfc..fc0aa6f235 100644
--- a/target/arm/gdbstub.c
+++ b/target/arm/gdbstub.c
@@ -25,11 +25,10 @@
 #include "internals.h"
 #include "cpregs.h"
 
-typedef struct RegisterSysregXmlParam {
+typedef struct RegisterSysregFeatureParam {
 CPUState *cs;
-GString *s;
-int n;
-} RegisterSysregXmlParam;
+GDBFeatureBuilder builder;
+} RegisterSysregFeatureParam;
 
 /* Old gdb always expect FPA registers.  Newer (xml-aware) gdb only expect
whatever the target description contains.  Due to a historical mishap
@@ -215,7 +214,7 @@ static int arm_gdb_get_sysreg(CPUARMState *env, GByteArray 
*buf, int reg)
 const ARMCPRegInfo *ri;
 uint32_t key;
 
-key = cpu->dyn_sysreg_xml.data.cpregs.keys[reg];
+key = cpu->dyn_sysreg_feature.data.cpregs.keys[reg];
 ri = get_arm_cp_reginfo(cpu->cp_regs, key);
 if (ri) {
 if (cpreg_field_is_64bit(ri)) {
@@ -232,34 +231,32 @@ static int arm_gdb_set_sysreg(CPUARMState *env, uint8_t 
*buf, int reg)
 return 0;
 }
 
-static void arm_gen_one_xml_sysreg_tag(GString *s, DynamicGDBXMLInfo *dyn_xml,
+static void arm_gen_one_feature_sysreg(GDBFeatureBuilder *builder,
+   DynamicGDBFeatureInfo *dyn_feature,
ARMCPRegInfo *ri, uint32_t ri_key,
-   int bitsize, int regnum)
+   int bitsize)
 {
-g_string_append_printf(s, "name);
-g_string_append_printf(s, " bitsize=\"%d\"", bitsize);
-g_string_append_printf(s, " regnum=\"%d\"", regnum);
-g_string_append_printf(s, " group=\"cp_regs\"/>");
-dyn_xml->data.cpregs.keys[dyn_xml->num] = ri_key;
-dyn_xml->num++;
+int num = gdb_feature_builder_append_reg(builder, ri->name, bitsize,
+ "int", "cp_regs");
+
+dyn_feature->data.cpregs.keys[num] = ri_key;
 }
 
-static void arm_register_sysreg_for_xml(gpointer key, gpointer value,
-gpointer p)
+static void arm_regist

[PATCH v9 11/23] gdbstub: Use GDBFeature for GDBRegisterState

[Akihiko Odaki]

Simplify GDBRegisterState by replacing num_regs and xml members with
one member that points to GDBFeature.

Signed-off-by: Akihiko Odaki 
---
 gdbstub/gdbstub.c | 14 ++
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index 20586e0b6a..9a4124afc4 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -47,10 +47,9 @@
 
 typedef struct GDBRegisterState {
 int base_reg;
-int num_regs;
 gdb_get_reg_cb get_reg;
 gdb_set_reg_cb set_reg;
-const char *xml;
+const GDBFeature *feature;
 } GDBRegisterState;
 
 GDBState gdbserver_state;
@@ -390,7 +389,7 @@ static const char *get_feature_xml(const char *p, const 
char **newp,
 g_ptr_array_add(
 xml,
 g_markup_printf_escaped("",
-r->xml));
+r->feature->xmlname));
 }
 g_ptr_array_add(xml, g_strdup(""));
 g_ptr_array_add(xml, NULL);
@@ -504,7 +503,7 @@ static int gdb_read_register(CPUState *cpu, GByteArray 
*buf, int reg)
 
 for (guint i = 0; i < cpu->gdb_regs->len; i++) {
 r = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
-if (r->base_reg <= reg && reg < r->base_reg + r->num_regs) {
+if (r->base_reg <= reg && reg < r->base_reg + r->feature->num_regs) {
 return r->get_reg(env, buf, reg - r->base_reg);
 }
 }
@@ -523,7 +522,7 @@ static int gdb_write_register(CPUState *cpu, uint8_t 
*mem_buf, int reg)
 
 for (guint i = 0; i < cpu->gdb_regs->len; i++) {
 r =  &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
-if (r->base_reg <= reg && reg < r->base_reg + r->num_regs) {
+if (r->base_reg <= reg && reg < r->base_reg + r->feature->num_regs) {
 return r->set_reg(env, mem_buf, reg - r->base_reg);
 }
 }
@@ -541,7 +540,7 @@ void gdb_register_coprocessor(CPUState *cpu,
 for (i = 0; i < cpu->gdb_regs->len; i++) {
 /* Check for duplicates.  */
 s = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
-if (strcmp(s->xml, feature->xmlname) == 0) {
+if (s->feature == feature) {
 return;
 }
 }
@@ -553,10 +552,9 @@ void gdb_register_coprocessor(CPUState *cpu,
 g_array_set_size(cpu->gdb_regs, i + 1);
 s = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
 s->base_reg = cpu->gdb_num_regs;
-s->num_regs = feature->num_regs;
 s->get_reg = get_reg;
 s->set_reg = set_reg;
-s->xml = feature->xml;
+s->feature = feature;
 
 /* Add to end of list.  */
 cpu->gdb_num_regs += feature->num_regs;
-- 
2.42.0

[PATCH v9 13/23] gdbstub: Simplify XML lookup

[Akihiko Odaki]

Now we know all instances of GDBFeature that is used in CPU so we can
traverse them to find XML. This removes the need for a CPU-specific
lookup function for dynamic XMLs.

Signed-off-by: Akihiko Odaki 
---
 include/exec/gdbstub.h |  2 +
 gdbstub/gdbstub.c  | 85 +++---
 hw/core/cpu-common.c   |  5 ++-
 3 files changed, 52 insertions(+), 40 deletions(-)

diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index 38b8db29f7..67f363132e 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -26,6 +26,8 @@ typedef struct GDBFeatureBuilder {
 typedef int (*gdb_get_reg_cb)(CPUState *cpu, GByteArray *buf, int reg);
 typedef int (*gdb_set_reg_cb)(CPUState *cpu, uint8_t *buf, int reg);
 
+void gdb_init_cpu(CPUState *cpu);
+
 /**
  * gdb_register_coprocessor() - register a supplemental set of registers
  * @cpu - the CPU associated with registers
diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index 63c049745a..49fb23a68a 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -352,6 +352,7 @@ static const char *get_feature_xml(const char *p, const 
char **newp,
 {
 CPUState *cpu = gdb_get_first_cpu_in_process(process);
 CPUClass *cc = CPU_GET_CLASS(cpu);
+GDBRegisterState *r;
 size_t len;
 
 /*
@@ -365,7 +366,6 @@ static const char *get_feature_xml(const char *p, const 
char **newp,
 /* Is it the main target xml? */
 if (strncmp(p, "target.xml", len) == 0) {
 if (!process->target_xml) {
-GDBRegisterState *r;
 g_autoptr(GPtrArray) xml = g_ptr_array_new_with_free_func(g_free);
 
 g_ptr_array_add(
@@ -380,10 +380,6 @@ static const char *get_feature_xml(const char *p, const 
char **newp,
 g_markup_printf_escaped("%s",
 cc->gdb_arch_name(cpu)));
 }
-g_ptr_array_add(
-xml,
-g_markup_printf_escaped("",
-cc->gdb_core_xml_file));
 for (guint i = 0; i < cpu->gdb_regs->len; i++) {
 r = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
 g_ptr_array_add(
@@ -398,20 +394,11 @@ static const char *get_feature_xml(const char *p, const 
char **newp,
 }
 return process->target_xml;
 }
-/* Is it dynamically generated by the target? */
-if (cc->gdb_get_dynamic_xml) {
-g_autofree char *xmlname = g_strndup(p, len);
-const char *xml = cc->gdb_get_dynamic_xml(cpu, xmlname);
-if (xml) {
-return xml;
-}
-}
-/* Is it one of the encoded gdb-xml/ files? */
-for (int i = 0; gdb_static_features[i].xmlname; i++) {
-const char *name = gdb_static_features[i].xmlname;
-if ((strncmp(name, p, len) == 0) &&
-strlen(name) == len) {
-return gdb_static_features[i].xml;
+/* Is it one of the features? */
+for (guint i = 0; i < cpu->gdb_regs->len; i++) {
+r = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
+if (strncmp(p, r->feature->xmlname, len) == 0) {
+return r->feature->xml;
 }
 }
 
@@ -527,40 +514,62 @@ static int gdb_write_register(CPUState *cpu, uint8_t 
*mem_buf, int reg)
 return 0;
 }
 
+static void gdb_register_feature(CPUState *cpu, int base_reg,
+ gdb_get_reg_cb get_reg, gdb_set_reg_cb 
set_reg,
+ const GDBFeature *feature)
+{
+guint i = cpu->gdb_regs->len;
+GDBRegisterState *s;
+
+g_array_set_size(cpu->gdb_regs, i + 1);
+s = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
+s->base_reg = base_reg;
+s->get_reg = get_reg;
+s->set_reg = set_reg;
+s->feature = feature;
+}
+
+void gdb_init_cpu(CPUState *cpu)
+{
+CPUClass *cc = CPU_GET_CLASS(cpu);
+const GDBFeature *feature;
+
+cpu->gdb_regs = g_array_new(false, false, sizeof(GDBRegisterState));
+
+if (cc->gdb_core_xml_file) {
+feature = gdb_find_static_feature(cc->gdb_core_xml_file);
+gdb_register_feature(cpu, 0,
+ cc->gdb_read_register, cc->gdb_write_register,
+ feature);
+}
+
+cpu->gdb_num_regs = cpu->gdb_num_g_regs = cc->gdb_num_core_regs;
+}
+
 void gdb_register_coprocessor(CPUState *cpu,
   gdb_get_reg_cb get_reg, gdb_set_reg_cb set_reg,
   const GDBFeature *feature, int g_pos)
 {
 GDBRegisterState *s;
 guint i;
+int base_reg = cpu->gdb_num_regs;
 
-if (cpu->gdb_regs) {
-for (i = 0; i < cpu->gdb_regs->len; i++) {
-/* Check for duplicates.  */
-s = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
-if (s->feature == feature) {
-return;
-}
+for (i = 0; i < cpu->gdb_regs->len; i++) {
+/* Check for duplicates.  */
+s = &g_array_ind

[PATCH v9 16/23] gdbstub: Add members to identify registers to GDBFeature

[Akihiko Odaki]

These members will be used to help plugins to identify registers.
The added members in instances of GDBFeature dynamically generated by
CPUs will be filled in later changes.

Signed-off-by: Akihiko Odaki 
---
 include/exec/gdbstub.h  |  3 +++
 gdbstub/gdbstub.c   | 10 --
 target/riscv/gdbstub.c  |  4 +---
 scripts/feature_to_c.py | 14 +-
 4 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index 67f363132e..b2b2675f6f 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -13,12 +13,15 @@
 typedef struct GDBFeature {
 const char *xmlname;
 const char *xml;
+const char *name;
+const char * const *regs;
 int num_regs;
 } GDBFeature;
 
 typedef struct GDBFeatureBuilder {
 GDBFeature *feature;
 GPtrArray *xml;
+GPtrArray *regs;
 } GDBFeatureBuilder;
 
 
diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index c19f1785e4..987af2298c 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -417,9 +417,10 @@ void gdb_feature_builder_init(GDBFeatureBuilder *builder, 
GDBFeature *feature,
 
 builder->feature = feature;
 builder->xml = g_ptr_array_new();
+builder->regs = g_ptr_array_new();
 g_ptr_array_add(builder->xml, header);
 feature->xmlname = xmlname;
-feature->num_regs = 0;
+feature->name = name;
 }
 
 void gdb_feature_builder_append_tag(const GDBFeatureBuilder *builder,
@@ -448,7 +449,9 @@ int gdb_feature_builder_append_reg(const GDBFeatureBuilder 
*builder,
 name, bitsize, type);
 }
 
-return builder->feature->num_regs++;
+g_ptr_array_add(builder->regs, (void *)name);
+
+return builder->regs->len - 1;
 }
 
 void gdb_feature_builder_end(const GDBFeatureBuilder *builder)
@@ -463,6 +466,9 @@ void gdb_feature_builder_end(const GDBFeatureBuilder 
*builder)
 }
 
 g_ptr_array_free(builder->xml, TRUE);
+
+builder->feature->num_regs = builder->regs->len;
+builder->feature->regs = (void *)g_ptr_array_free(builder->regs, FALSE);
 }
 
 const GDBFeature *gdb_find_static_feature(const char *xmlname)
diff --git a/target/riscv/gdbstub.c b/target/riscv/gdbstub.c
index aa0824e119..9082b0c6e2 100644
--- a/target/riscv/gdbstub.c
+++ b/target/riscv/gdbstub.c
@@ -265,11 +265,9 @@ static GDBFeature *riscv_gen_dynamic_csr_feature(CPUState 
*cs)
 }
 predicate = csr_ops[i].predicate;
 if (predicate && (predicate(env, i) == RISCV_EXCP_NONE)) {
-g_autofree char *dynamic_name = NULL;
 name = csr_ops[i].name;
 if (!name) {
-dynamic_name = g_strdup_printf("csr%03x", i);
-name = dynamic_name;
+name = g_strdup_printf("csr%03x", i);
 }
 
 gdb_feature_builder_append_reg(&builder, name, bitsize,
diff --git a/scripts/feature_to_c.py b/scripts/feature_to_c.py
index e04d6b2df7..807af0e685 100755
--- a/scripts/feature_to_c.py
+++ b/scripts/feature_to_c.py
@@ -50,7 +50,9 @@ def writeliteral(indent, bytes):
 sys.stderr.write(f'unexpected start tag: {element.tag}\n')
 exit(1)
 
+feature_name = element.attrib['name']
 regnum = 0
+regnames = []
 regnums = []
 tags = ['feature']
 for event, element in events:
@@ -67,6 +69,7 @@ def writeliteral(indent, bytes):
 if 'regnum' in element.attrib:
 regnum = int(element.attrib['regnum'])
 
+regnames.append(element.attrib['name'])
 regnums.append(regnum)
 regnum += 1
 
@@ -85,6 +88,15 @@ def writeliteral(indent, bytes):
 writeliteral(8, bytes(os.path.basename(input), 'utf-8'))
 sys.stdout.write(',\n')
 writeliteral(8, read)
-sys.stdout.write(f',\n{num_regs},\n}},\n')
+sys.stdout.write(',\n')
+writeliteral(8, bytes(feature_name, 'utf-8'))
+sys.stdout.write(',\n(const char * const []) {\n')
+
+for index, regname in enumerate(regnames):
+sys.stdout.write(f'[{regnums[index] - base_reg}] =\n')
+writeliteral(16, bytes(regname, 'utf-8'))
+sys.stdout.write(',\n')
+
+sys.stdout.write(f'}},\n{num_regs},\n}},\n')
 
 sys.stdout.write('{ NULL }\n};\n')
-- 
2.42.0

[PATCH v9 04/23] gdbstub: Add num_regs member to GDBFeature

[Akihiko Odaki]

Currently the number of registers exposed to GDB is written as magic
numbers in code. Derive the number of registers GDB actually see from
XML files to replace the magic numbers in code later.

Signed-off-by: Akihiko Odaki 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Alex Bennée 
---
 include/exec/gdbstub.h  |  1 +
 scripts/feature_to_c.py | 46 +++--
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index 1a01c35f8e..a43aa34dad 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -13,6 +13,7 @@
 typedef struct GDBFeature {
 const char *xmlname;
 const char *xml;
+int num_regs;
 } GDBFeature;
 
 
diff --git a/scripts/feature_to_c.py b/scripts/feature_to_c.py
index bcbcb83beb..e04d6b2df7 100755
--- a/scripts/feature_to_c.py
+++ b/scripts/feature_to_c.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os, sys
+import os, sys, xml.etree.ElementTree
 
 def writeliteral(indent, bytes):
 sys.stdout.write(' ' * indent)
@@ -39,10 +39,52 @@ def writeliteral(indent, bytes):
 with open(input, 'rb') as file:
 read = file.read()
 
+parser = xml.etree.ElementTree.XMLPullParser(['start', 'end'])
+parser.feed(read)
+events = parser.read_events()
+event, element = next(events)
+if event != 'start':
+sys.stderr.write(f'unexpected event: {event}\n')
+exit(1)
+if element.tag != 'feature':
+sys.stderr.write(f'unexpected start tag: {element.tag}\n')
+exit(1)
+
+regnum = 0
+regnums = []
+tags = ['feature']
+for event, element in events:
+if event == 'end':
+if element.tag != tags[len(tags) - 1]:
+sys.stderr.write(f'unexpected end tag: {element.tag}\n')
+exit(1)
+
+tags.pop()
+if element.tag == 'feature':
+break
+elif event == 'start':
+if len(tags) < 2 and element.tag == 'reg':
+if 'regnum' in element.attrib:
+regnum = int(element.attrib['regnum'])
+
+regnums.append(regnum)
+regnum += 1
+
+tags.append(element.tag)
+else:
+raise Exception(f'unexpected event: {event}\n')
+
+if len(tags):
+sys.stderr.write('unterminated feature tag\n')
+exit(1)
+
+base_reg = min(regnums)
+num_regs = max(regnums) - base_reg + 1 if len(regnums) else 0
+
 sys.stdout.write('{\n')
 writeliteral(8, bytes(os.path.basename(input), 'utf-8'))
 sys.stdout.write(',\n')
 writeliteral(8, read)
-sys.stdout.write('\n},\n')
+sys.stdout.write(f',\n{num_regs},\n}},\n')
 
 sys.stdout.write('{ NULL }\n};\n')
-- 
2.42.0

[PATCH v9 23/23] plugins: Support C++

[Akihiko Odaki]

Make qemu-plugin.h consumable for C++ platform.

Signed-off-by: Akihiko Odaki 
---
 docs/devel/tcg-plugins.rst |  4 
 meson.build|  2 +-
 include/qemu/qemu-plugin.h |  4 
 tests/plugin/cc.cc | 16 
 tests/plugin/meson.build   |  5 +
 tests/tcg/Makefile.target  |  3 +--
 6 files changed, 31 insertions(+), 3 deletions(-)
 create mode 100644 tests/plugin/cc.cc

diff --git a/docs/devel/tcg-plugins.rst b/docs/devel/tcg-plugins.rst
index c9f8b27590..984d8012e9 100644
--- a/docs/devel/tcg-plugins.rst
+++ b/docs/devel/tcg-plugins.rst
@@ -283,6 +283,10 @@ run::
   160  1  0
   135  1  0
 
+- contrib/plugins/cc.cc
+
+A pure test plugin to ensure that the plugin API is compatible with C++.
+
 - contrib/plugins/hotblocks.c
 
 The hotblocks plugin allows you to examine the where hot paths of
diff --git a/meson.build b/meson.build
index 9f4c4f2f1e..c289bb8948 100644
--- a/meson.build
+++ b/meson.build
@@ -20,7 +20,7 @@ config_host = keyval.load(meson.current_build_dir() / 
'config-host.mak')
 
 cc = meson.get_compiler('c')
 all_languages = ['c']
-if targetos == 'windows' and add_languages('cpp', required: false, native: 
false)
+if add_languages('cpp', required: false, native: false)
   all_languages += ['cpp']
   cxx = meson.get_compiler('cpp')
 endif
diff --git a/include/qemu/qemu-plugin.h b/include/qemu/qemu-plugin.h
index 40aae8db68..55f514ca6c 100644
--- a/include/qemu/qemu-plugin.h
+++ b/include/qemu/qemu-plugin.h
@@ -16,6 +16,8 @@
 #include 
 #include 
 
+G_BEGIN_DECLS
+
 /*
  * For best performance, build the plugin with -fvisibility=hidden so that
  * QEMU_PLUGIN_LOCAL is implicit. Then, just mark qemu_plugin_install with
@@ -710,4 +712,6 @@ int qemu_plugin_find_register(unsigned int vcpu_index, int 
file,
  */
 int qemu_plugin_read_register(GByteArray *buf, int reg);
 
+G_END_DECLS
+
 #endif /* QEMU_QEMU_PLUGIN_H */
diff --git a/tests/plugin/cc.cc b/tests/plugin/cc.cc
new file mode 100644
index 00..297a7e4f3f
--- /dev/null
+++ b/tests/plugin/cc.cc
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include 
+
+extern "C" {
+
+QEMU_PLUGIN_EXPORT int qemu_plugin_version = QEMU_PLUGIN_VERSION;
+
+QEMU_PLUGIN_EXPORT int qemu_plugin_install(qemu_plugin_id_t id,
+   const qemu_info_t *info, int argc,
+   char **argv)
+{
+return 0;
+}
+
+};
diff --git a/tests/plugin/meson.build b/tests/plugin/meson.build
index 322cafcdf6..fed14aa0c2 100644
--- a/tests/plugin/meson.build
+++ b/tests/plugin/meson.build
@@ -5,6 +5,11 @@ if get_option('plugins')
include_directories: '../../include/qemu',
dependencies: glib)
   endforeach
+  if 'cpp' in all_languages
+t += shared_module('cc', files('cc.cc'),
+   include_directories: '../../include/qemu',
+   dependencies: glib)
+  endif
 endif
 if t.length() > 0
   alias_target('test-plugins', t)
diff --git a/tests/tcg/Makefile.target b/tests/tcg/Makefile.target
index 462289f47c..66a20d0f2c 100644
--- a/tests/tcg/Makefile.target
+++ b/tests/tcg/Makefile.target
@@ -145,10 +145,9 @@ RUN_TESTS=$(patsubst %,run-%, $(TESTS))
 
 # If plugins exist also include those in the tests
 ifeq ($(CONFIG_PLUGIN),y)
-PLUGIN_SRC=$(SRC_PATH)/tests/plugin
 PLUGIN_LIB=../../plugin
 VPATH+=$(PLUGIN_LIB)
-PLUGINS=$(patsubst %.c, lib%.so, $(notdir $(wildcard $(PLUGIN_SRC)/*.c)))
+PLUGINS=$(notdir $(wildcard $(PLUGIN_LIB)/*.so))
 
 # We need to ensure expand the run-plugin-TEST-with-PLUGIN
 # pre-requistes manually here as we can't use stems to handle it. We
-- 
2.42.0

[PATCH v9 22/23] contrib/plugins: Allow to log registers

[Akihiko Odaki]

This demonstrates how a register can be read from a plugin.

Signed-off-by: Akihiko Odaki 
---
 docs/devel/tcg-plugins.rst |  10 +++-
 contrib/plugins/execlog.c  | 120 +++--
 2 files changed, 97 insertions(+), 33 deletions(-)

diff --git a/docs/devel/tcg-plugins.rst b/docs/devel/tcg-plugins.rst
index 81dcd43a61..c9f8b27590 100644
--- a/docs/devel/tcg-plugins.rst
+++ b/docs/devel/tcg-plugins.rst
@@ -497,6 +497,15 @@ arguments if required::
   $ qemu-system-arm $(QEMU_ARGS) \
 -plugin ./contrib/plugins/libexeclog.so,ifilter=st1w,afilter=0x40001808 -d 
plugin
 
+This plugin can also dump a specified register. The specification of register
+follows `GDB standard target features 
`__.
+
+Specify the name of the feature that contains the register and the name of the
+register with ``rfile`` and ``reg`` options, respectively::
+
+  $ qemu-system-arm $(QEMU_ARGS) \
+-plugin ./contrib/plugins/libexeclog.so,rfile=org.gnu.gdb.arm.core,reg=sp 
-d plugin
+
 - contrib/plugins/cache.c
 
 Cache modelling plugin that measures the performance of a given L1 cache
@@ -583,4 +592,3 @@ The following API is generated from the inline 
documentation in
 include the full kernel-doc annotations.
 
 .. kernel-doc:: include/qemu/qemu-plugin.h
-
diff --git a/contrib/plugins/execlog.c b/contrib/plugins/execlog.c
index 82dc2f584e..f3e714c888 100644
--- a/contrib/plugins/execlog.c
+++ b/contrib/plugins/execlog.c
@@ -15,27 +15,43 @@
 
 #include 
 
+typedef struct CPU {
+/* Store last executed instruction on each vCPU as a GString */
+GString *last_exec;
+GByteArray *reg_history[2];
+
+int reg;
+} CPU;
+
 QEMU_PLUGIN_EXPORT int qemu_plugin_version = QEMU_PLUGIN_VERSION;
 
-/* Store last executed instruction on each vCPU as a GString */
-static GPtrArray *last_exec;
+static CPU *cpus;
+static int num_cpus;
 static GRWLock expand_array_lock;
 
 static GPtrArray *imatches;
 static GArray *amatches;
 
+static char *rfile_name;
+static char *reg_name;
+
 /*
- * Expand last_exec array.
+ * Expand cpu array.
  *
  * As we could have multiple threads trying to do this we need to
  * serialise the expansion under a lock.
  */
-static void expand_last_exec(int cpu_index)
+static void expand_cpu(int cpu_index)
 {
 g_rw_lock_writer_lock(&expand_array_lock);
-while (cpu_index >= last_exec->len) {
-GString *s = g_string_new(NULL);
-g_ptr_array_add(last_exec, s);
+if (cpu_index >= num_cpus) {
+cpus = g_realloc_n(cpus, cpu_index + 1, sizeof(*cpus));
+while (cpu_index >= num_cpus) {
+cpus[num_cpus].last_exec = g_string_new(NULL);
+cpus[num_cpus].reg_history[0] = g_byte_array_new();
+cpus[num_cpus].reg_history[1] = g_byte_array_new();
+num_cpus++;
+}
 }
 g_rw_lock_writer_unlock(&expand_array_lock);
 }
@@ -50,8 +66,8 @@ static void vcpu_mem(unsigned int cpu_index, 
qemu_plugin_meminfo_t info,
 
 /* Find vCPU in array */
 g_rw_lock_reader_lock(&expand_array_lock);
-g_assert(cpu_index < last_exec->len);
-s = g_ptr_array_index(last_exec, cpu_index);
+g_assert(cpu_index < num_cpus);
+s = cpus[cpu_index].last_exec;
 g_rw_lock_reader_unlock(&expand_array_lock);
 
 /* Indicate type of memory access */
@@ -77,28 +93,42 @@ static void vcpu_mem(unsigned int cpu_index, 
qemu_plugin_meminfo_t info,
  */
 static void vcpu_insn_exec(unsigned int cpu_index, void *udata)
 {
-GString *s;
+int n;
+int i;
 
-/* Find or create vCPU in array */
 g_rw_lock_reader_lock(&expand_array_lock);
-if (cpu_index >= last_exec->len) {
-g_rw_lock_reader_unlock(&expand_array_lock);
-expand_last_exec(cpu_index);
-g_rw_lock_reader_lock(&expand_array_lock);
-}
-s = g_ptr_array_index(last_exec, cpu_index);
-g_rw_lock_reader_unlock(&expand_array_lock);
 
 /* Print previous instruction in cache */
-if (s->len) {
-qemu_plugin_outs(s->str);
+if (cpus[cpu_index].last_exec->len) {
+if (cpus[cpu_index].reg >= 0) {
+GByteArray *current = cpus[cpu_index].reg_history[0];
+GByteArray *last = cpus[cpu_index].reg_history[1];
+
+g_byte_array_set_size(current, 0);
+n = qemu_plugin_read_register(current, cpus[cpu_index].reg);
+
+if (n != last->len || memcmp(current->data, last->data, n)) {
+g_string_append(cpus[cpu_index].last_exec, ", reg,");
+for (i = 0; i < n; i++) {
+g_string_append_printf(cpus[cpu_index].last_exec, " %02x",
+   current->data[i]);
+}
+}
+
+cpus[cpu_index].reg_history[0] = last;
+cpus[cpu_index].reg_history[1] = current;
+}
+
+qemu_plugin_outs(cpus[cpu_index].last_exec->str);
 qemu_plugin_outs("\n");
 }
 
 /* St

[PATCH v9 12/23] gdbstub: Change gdb_get_reg_cb and gdb_set_reg_cb

[Akihiko Odaki]

Align the parameters of gdb_get_reg_cb and gdb_set_reg_cb with the
gdb_read_register and gdb_write_register members of CPUClass to allow
to unify the logic to access registers of the core and coprocessors
in the future.

Signed-off-by: Akihiko Odaki 
---
 include/exec/gdbstub.h  |  4 +-
 target/arm/internals.h  | 12 +++---
 target/hexagon/internal.h   |  4 +-
 target/microblaze/cpu.h |  4 +-
 gdbstub/gdbstub.c   |  6 +--
 target/arm/gdbstub.c| 51 
 target/arm/gdbstub64.c  | 27 +
 target/hexagon/gdbstub.c| 10 -
 target/loongarch/gdbstub.c  | 11 --
 target/m68k/helper.c| 20 --
 target/microblaze/gdbstub.c |  9 -
 target/ppc/gdbstub.c| 46 +-
 target/riscv/gdbstub.c  | 46 --
 target/s390x/gdbstub.c  | 77 -
 14 files changed, 236 insertions(+), 91 deletions(-)

diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index 5525eea090..38b8db29f7 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -23,8 +23,8 @@ typedef struct GDBFeatureBuilder {
 
 
 /* Get or set a register.  Returns the size of the register.  */
-typedef int (*gdb_get_reg_cb)(CPUArchState *env, GByteArray *buf, int reg);
-typedef int (*gdb_set_reg_cb)(CPUArchState *env, uint8_t *buf, int reg);
+typedef int (*gdb_get_reg_cb)(CPUState *cpu, GByteArray *buf, int reg);
+typedef int (*gdb_set_reg_cb)(CPUState *cpu, uint8_t *buf, int reg);
 
 /**
  * gdb_register_coprocessor() - register a supplemental set of registers
diff --git a/target/arm/internals.h b/target/arm/internals.h
index 09350e96da..f7a48bad5f 100644
--- a/target/arm/internals.h
+++ b/target/arm/internals.h
@@ -1441,12 +1441,12 @@ static inline uint64_t pmu_counter_mask(CPUARMState 
*env)
 
 #ifdef TARGET_AARCH64
 GDBFeature *arm_gen_dynamic_svereg_feature(CPUState *cpu);
-int aarch64_gdb_get_sve_reg(CPUARMState *env, GByteArray *buf, int reg);
-int aarch64_gdb_set_sve_reg(CPUARMState *env, uint8_t *buf, int reg);
-int aarch64_gdb_get_fpu_reg(CPUARMState *env, GByteArray *buf, int reg);
-int aarch64_gdb_set_fpu_reg(CPUARMState *env, uint8_t *buf, int reg);
-int aarch64_gdb_get_pauth_reg(CPUARMState *env, GByteArray *buf, int reg);
-int aarch64_gdb_set_pauth_reg(CPUARMState *env, uint8_t *buf, int reg);
+int aarch64_gdb_get_sve_reg(CPUState *cs, GByteArray *buf, int reg);
+int aarch64_gdb_set_sve_reg(CPUState *cs, uint8_t *buf, int reg);
+int aarch64_gdb_get_fpu_reg(CPUState *cs, GByteArray *buf, int reg);
+int aarch64_gdb_set_fpu_reg(CPUState *cs, uint8_t *buf, int reg);
+int aarch64_gdb_get_pauth_reg(CPUState *cs, GByteArray *buf, int reg);
+int aarch64_gdb_set_pauth_reg(CPUState *cs, uint8_t *buf, int reg);
 void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp);
 void arm_cpu_sme_finalize(ARMCPU *cpu, Error **errp);
 void arm_cpu_pauth_finalize(ARMCPU *cpu, Error **errp);
diff --git a/target/hexagon/internal.h b/target/hexagon/internal.h
index d732b6bb3c..beb08cb7e3 100644
--- a/target/hexagon/internal.h
+++ b/target/hexagon/internal.h
@@ -33,8 +33,8 @@
 
 int hexagon_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg);
 int hexagon_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-int hexagon_hvx_gdb_read_register(CPUHexagonState *env, GByteArray *mem_buf, 
int n);
-int hexagon_hvx_gdb_write_register(CPUHexagonState *env, uint8_t *mem_buf, int 
n);
+int hexagon_hvx_gdb_read_register(CPUState *env, GByteArray *mem_buf, int n);
+int hexagon_hvx_gdb_write_register(CPUState *env, uint8_t *mem_buf, int n);
 
 void hexagon_debug_vreg(CPUHexagonState *env, int regnum);
 void hexagon_debug_qreg(CPUHexagonState *env, int regnum);
diff --git a/target/microblaze/cpu.h b/target/microblaze/cpu.h
index e43c49d4af..c14b3357b7 100644
--- a/target/microblaze/cpu.h
+++ b/target/microblaze/cpu.h
@@ -370,8 +370,8 @@ G_NORETURN void mb_cpu_do_unaligned_access(CPUState *cs, 
vaddr vaddr,
 void mb_cpu_dump_state(CPUState *cpu, FILE *f, int flags);
 int mb_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg);
 int mb_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-int mb_cpu_gdb_read_stack_protect(CPUArchState *cpu, GByteArray *buf, int reg);
-int mb_cpu_gdb_write_stack_protect(CPUArchState *cpu, uint8_t *buf, int reg);
+int mb_cpu_gdb_read_stack_protect(CPUState *cs, GByteArray *buf, int reg);
+int mb_cpu_gdb_write_stack_protect(CPUState *cs, uint8_t *buf, int reg);
 
 static inline uint32_t mb_cpu_read_msr(const CPUMBState *env)
 {
diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index 9a4124afc4..63c049745a 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -494,7 +494,6 @@ const GDBFeature *gdb_find_static_feature(const char 
*xmlname)
 static int gdb_read_register(CPUState *cpu, GByteArray *buf, int reg)
 {
 CPUClass *cc = CPU_GET_CLASS(cpu);
-CPUArchState *env = cpu_env(cpu);
 GDBRegisterState *r;
 
 if (reg < cc->gdb_num_core_

[PATCH v9 19/23] plugins: Remove an extra parameter

[Akihiko Odaki]

copy_call() has an unused parameter so remove it.

Signed-off-by: Akihiko Odaki 
---
 accel/tcg/plugin-gen.c | 9 +++--
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c
index 39b3c9351f..78b331b251 100644
--- a/accel/tcg/plugin-gen.c
+++ b/accel/tcg/plugin-gen.c
@@ -327,8 +327,7 @@ static TCGOp *copy_st_ptr(TCGOp **begin_op, TCGOp *op)
 return op;
 }
 
-static TCGOp *copy_call(TCGOp **begin_op, TCGOp *op, void *empty_func,
-void *func, int *cb_idx)
+static TCGOp *copy_call(TCGOp **begin_op, TCGOp *op, void *func, int *cb_idx)
 {
 TCGOp *old_op;
 int func_idx;
@@ -372,8 +371,7 @@ static TCGOp *append_udata_cb(const struct 
qemu_plugin_dyn_cb *cb,
 }
 
 /* call */
-op = copy_call(&begin_op, op, HELPER(plugin_vcpu_udata_cb),
-   cb->f.vcpu_udata, cb_idx);
+op = copy_call(&begin_op, op, cb->f.vcpu_udata, cb_idx);
 
 return op;
 }
@@ -420,8 +418,7 @@ static TCGOp *append_mem_cb(const struct qemu_plugin_dyn_cb 
*cb,
 
 if (type == PLUGIN_GEN_CB_MEM) {
 /* call */
-op = copy_call(&begin_op, op, HELPER(plugin_vcpu_mem_cb),
-   cb->f.vcpu_udata, cb_idx);
+op = copy_call(&begin_op, op, cb->f.vcpu_udata, cb_idx);
 }
 
 return op;
-- 
2.42.0

[PATCH v9 17/23] gdbstub: Expose functions to read registers

[Akihiko Odaki]

gdb_find_feature() and gdb_find_feature_register() find registers.
gdb_read_register() actually reads registers.

Signed-off-by: Akihiko Odaki 
---
 include/exec/gdbstub.h |  5 +
 gdbstub/gdbstub.c  | 31 ++-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index b2b2675f6f..d73abfbd44 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -71,6 +71,11 @@ void gdb_feature_builder_end(const GDBFeatureBuilder 
*builder);
 
 const GDBFeature *gdb_find_static_feature(const char *xmlname);
 
+int gdb_find_feature(CPUState *cpu, const char *name);
+int gdb_find_feature_register(CPUState *cpu, int feature, const char *name);
+
+int gdb_read_register(CPUState *cpu, GByteArray *buf, int reg);
+
 void gdb_set_stop_cpu(CPUState *cpu);
 
 /* in gdbstub-xml.c, generated by scripts/feature_to_c.py */
diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index 987af2298c..a3fc65ec86 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -484,7 +484,36 @@ const GDBFeature *gdb_find_static_feature(const char 
*xmlname)
 g_assert_not_reached();
 }
 
-static int gdb_read_register(CPUState *cpu, GByteArray *buf, int reg)
+int gdb_find_feature(CPUState *cpu, const char *name)
+{
+GDBRegisterState *r;
+
+for (guint i = 0; i < cpu->gdb_regs->len; i++) {
+r = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
+if (!strcmp(name, r->feature->name)) {
+return i;
+}
+}
+
+return -1;
+}
+
+int gdb_find_feature_register(CPUState *cpu, int feature, const char *name)
+{
+GDBRegisterState *r;
+
+r = &g_array_index(cpu->gdb_regs, GDBRegisterState, feature);
+
+for (int i = 0; i < r->feature->num_regs; i++) {
+if (r->feature->regs[i] && !strcmp(name, r->feature->regs[i])) {
+return r->base_reg + i;
+}
+}
+
+return -1;
+}
+
+int gdb_read_register(CPUState *cpu, GByteArray *buf, int reg)
 {
 CPUClass *cc = CPU_GET_CLASS(cpu);
 GDBRegisterState *r;
-- 
2.42.0

[PATCH v9 21/23] plugins: Allow to read registers

[Akihiko Odaki]

It is based on GDB protocol to ensure interface stability.

The timing of the vcpu init hook is also changed so that the hook will
get called after GDB features are initialized.

Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1706
Signed-off-by: Akihiko Odaki 
---
 include/qemu/qemu-plugin.h   | 52 +---
 plugins/api.c| 20 ++
 plugins/qemu-plugins.symbols |  3 +++
 3 files changed, 72 insertions(+), 3 deletions(-)

diff --git a/include/qemu/qemu-plugin.h b/include/qemu/qemu-plugin.h
index 50a9957279..40aae8db68 100644
--- a/include/qemu/qemu-plugin.h
+++ b/include/qemu/qemu-plugin.h
@@ -11,6 +11,7 @@
 #ifndef QEMU_QEMU_PLUGIN_H
 #define QEMU_QEMU_PLUGIN_H
 
+#include 
 #include 
 #include 
 #include 
@@ -51,7 +52,7 @@ typedef uint64_t qemu_plugin_id_t;
 
 extern QEMU_PLUGIN_EXPORT int qemu_plugin_version;
 
-#define QEMU_PLUGIN_VERSION 1
+#define QEMU_PLUGIN_VERSION 2
 
 /**
  * struct qemu_info_t - system information for plugins
@@ -218,8 +219,8 @@ struct qemu_plugin_insn;
  * @QEMU_PLUGIN_CB_R_REGS: callback reads the CPU's regs
  * @QEMU_PLUGIN_CB_RW_REGS: callback reads and writes the CPU's regs
  *
- * Note: currently unused, plugins cannot read or change system
- * register state.
+ * Note: currently QEMU_PLUGIN_CB_RW_REGS is unused, plugins cannot change
+ * system register state.
  */
 enum qemu_plugin_cb_flags {
 QEMU_PLUGIN_CB_NO_REGS,
@@ -664,4 +665,49 @@ uint64_t qemu_plugin_end_code(void);
  */
 uint64_t qemu_plugin_entry_code(void);
 
+/**
+ * qemu_plugin_find_register_file() - find register file
+ *
+ * @vcpu_index: the index of the vcpu context
+ * @name: the name of the register file.
+ *
+ * Returns the identifier of the register file if it was found, and a negative
+ * value otherwise.
+ *
+ * The names of register files are identical with names of GDB's standard
+ * target features with some extensions. For details, see:
+ * https://sourceware.org/gdb/onlinedocs/gdb/Standard-Target-Features.html
+ */
+int qemu_plugin_find_register_file(unsigned int vcpu_index, const char *name);
+
+/**
+ * qemu_plugin_find_register() - find register
+ *
+ * @vcpu_index: the index of the vcpu context
+ * @file: the register file identifier determined with
+ *qemu_plugin_find_register_file().
+ * @name: the name of the register.
+ *
+ * The names of register are identical with names used in GDB's standard
+ * target features with some extensions. For details, see:
+ * https://sourceware.org/gdb/onlinedocs/gdb/Standard-Target-Features.html
+ */
+int qemu_plugin_find_register(unsigned int vcpu_index, int file,
+  const char *name);
+
+/**
+ * qemu_plugin_read_register() - read register
+ *
+ * @buf: the byte array to append the read register content to.
+ * @reg: the register identifier determined with
+ *   qemu_plugin_find_register().
+ *
+ * This function is only available in a context that register read access is
+ * explicitly requested.
+ *
+ * Returns the size of the read register. The content of @buf is in target byte
+ * order.
+ */
+int qemu_plugin_read_register(GByteArray *buf, int reg);
+
 #endif /* QEMU_QEMU_PLUGIN_H */
diff --git a/plugins/api.c b/plugins/api.c
index 3f7b5bbfdd..6c7343edfe 100644
--- a/plugins/api.c
+++ b/plugins/api.c
@@ -35,10 +35,12 @@
  */
 
 #include "qemu/osdep.h"
+#include "qemu/main-loop.h"
 #include "qemu/plugin.h"
 #include "qemu/log.h"
 #include "tcg/tcg.h"
 #include "exec/exec-all.h"
+#include "exec/gdbstub.h"
 #include "exec/ram_addr.h"
 #include "disas/disas.h"
 #include "plugin.h"
@@ -433,3 +435,21 @@ uint64_t qemu_plugin_entry_code(void)
 #endif
 return entry;
 }
+
+int qemu_plugin_find_register_file(unsigned int vcpu_index, const char *name)
+{
+QEMU_IOTHREAD_LOCK_GUARD();
+return gdb_find_feature(qemu_get_cpu(vcpu_index), name);
+}
+
+int qemu_plugin_find_register(unsigned int vcpu_index, int file,
+  const char *name)
+{
+QEMU_IOTHREAD_LOCK_GUARD();
+return gdb_find_feature_register(qemu_get_cpu(vcpu_index), file, name);
+}
+
+int qemu_plugin_read_register(GByteArray *buf, int reg)
+{
+return gdb_read_register(current_cpu, buf, reg);
+}
diff --git a/plugins/qemu-plugins.symbols b/plugins/qemu-plugins.symbols
index 71f6c90549..977f1fcfcb 100644
--- a/plugins/qemu-plugins.symbols
+++ b/plugins/qemu-plugins.symbols
@@ -42,4 +42,7 @@
   qemu_plugin_tb_vaddr;
   qemu_plugin_uninstall;
   qemu_plugin_vcpu_for_each;
+  qemu_plugin_find_register_file;
+  qemu_plugin_find_register;
+  qemu_plugin_read_register;
 };
-- 
2.42.0

[PATCH v9 00/23] plugins: Allow to read registers

[Akihiko Odaki]

Based-on: <20231009164104.369749-4-alex.ben...@linaro.org>
("[PATCH 00/25] October maintainer omnibus pre-PR (tests, gdbstub,
plugins)")

I and other people in the University of Tokyo, where I research processor
design, found TCG plugins are very useful for processor design exploration.

The feature we find missing is the capability to read registers from
plugins. In this series, I propose to add such a capability by reusing
gdbstub code.

The reuse of gdbstub code ensures the long-term stability of the TCG plugin
interface for register access without incurring a burden to maintain yet
another interface for register access.

This process to add TCG plugin involves four major changes. The first one
is to add GDBFeature structure that represents a GDB feature, which usually
includes registers. GDBFeature can be generated from static XML files or
dynamically generated by architecture-specific code. In fact, this is a
refactoring independent of the feature this series adds, and potentially
it's benefitial even without the plugin feature. The plugin feature will
utilize this new structure to describe registers exposed to plugins.

The second one is to make gdb_read_register/gdb_write_register usable
outside of gdbstub context.

The third one is to actually make registers readable for plugins.

The last one is to allow to implement a QEMU plugin in C++. A plugin that
I'll describe later is written in C++.

The below is a summary of patches:
Patch [01, 15] introduces num_regs member to GDBFeature.
Patch 16 adds members useful to identify registers to GDBFeature.
Patch 17 makes registers readable outside of gdbstub context.
Patch [18, 22] add the feature to read registers from plugins.
Patch 23 make it possible to write plugins in C++.

V8 -> V9:
  Rebased to "[PATCH 00/25] October maintainer omnibus pre-PR (tests,
  gdbstub, plugins)".
  Added patch "target/riscv: Move MISA limits to class".
  Added patch "target/riscv: Remove misa_mxl validation".
  Added patch "target/riscv: Validate misa_mxl_max only once".
  Added patch "plugins: Use different helpers when reading".
  Moved contrib/plugins/cc.cc to tests/plugin/cc.cc.

V7 -> V8:
  Rebased to "[PATCH v3 00/12] gdbstub and TCG plugin improvements".
  Clarified that initialization and exit hooks affect TCG state.
  Simplified by adding the core feature to gdb_regs.

V6 -> V7:
  Rebased to "[PATCH v2 00/11] gdbstub and TCG plugin improvements".
  Replaced functions to get register identifiers.

V5 -> V6:
  Rebased to "[PATCH 0/8] gdbstub and TCG plugin improvements".

V4 -> V5:
  Corrected g_rw_lock_writer_lock() call. (Richard Henderson)
  Replaced abort() with g_assert_not_reached(). (Richard Henderson)
  Fixed CSR name leak in target/riscv. (Richard Henderson)
  Removed gdb_has_xml variable.

V3 -> V4:
  Added execlog changes I forgot to include in the last version.

V2 -> V3:
  Added patch "hw/core/cpu: Return static value with gdb_arch_name()".
  Added patch "gdbstub: Dynamically allocate target.xml buffer".
  (Alex Bennée)
  Added patch "gdbstub: Introduce GDBFeatureBuilder". (Alex Bennée)
  Dropped Reviewed-by tags for "target/*: Use GDBFeature for dynamic XML".
  Changed gdb_find_static_feature() to abort on failure. (Alex Bennée)
  Changed the execlog plugin to log the register value only when changed.
  (Alex Bennée)
  Dropped 0x prefixes for register value logs for conciseness.

V1 -> V2:
  Added SPDX-License-Identifier: GPL-2.0-or-later. (Philippe Mathieu-Daudé)
  Split long lines. (Philippe Mathieu-Daudé)
  Renamed gdb_features to gdb_static_features (Philippe Mathieu-Daudé)
  Dropped RFC.

The execlog plugin will have new options to demonstrate the new feature.
I also have a plugin that uses this new feature to generate execution
traces for Sniper processor simulator, which is available at:
https://github.com/shioya-lab/sniper/tree/akihikodaki/bb

Akihiko Odaki (23):
  target/riscv: Move MISA limits to class
  target/riscv: Remove misa_mxl validation
  target/riscv: Validate misa_mxl_max only once
  gdbstub: Add num_regs member to GDBFeature
  gdbstub: Introduce gdb_find_static_feature()
  gdbstub: Introduce GDBFeatureBuilder
  target/arm: Use GDBFeature for dynamic XML
  target/ppc: Use GDBFeature for dynamic XML
  target/riscv: Use GDBFeature for dynamic XML
  gdbstub: Use GDBFeature for gdb_register_coprocessor
  gdbstub: Use GDBFeature for GDBRegisterState
  gdbstub: Change gdb_get_reg_cb and gdb_set_reg_cb
  gdbstub: Simplify XML lookup
  gdbstub: Infer number of core registers from XML
  hw/core/cpu: Remove gdb_get_dynamic_xml member
  gdbstub: Add members to identify registers to GDBFeature
  gdbstub: Expose functions to read registers
  cpu: Call plugin hooks only when ready
  plugins: Remove an extra parameter
  plugins: Use different helpers when reading registers
  plugins: Allow to read registers
  contrib/plugins: Allow to log registers
  plugins: Support C++

 docs/devel/tcg-plugins.rst   |  14 +-
 meson.build  |   2 +-

[PATCH v9 10/23] gdbstub: Use GDBFeature for gdb_register_coprocessor

[Akihiko Odaki]

This is a tree-wide change to introduce GDBFeature parameter to
gdb_register_coprocessor(). The new parameter just replaces num_regs
and xml parameters for now. GDBFeature will be utilized to simplify XML
lookup in a following change.

Signed-off-by: Akihiko Odaki 
Acked-by: Alex Bennée 
---
 include/exec/gdbstub.h |  2 +-
 gdbstub/gdbstub.c  | 13 +++--
 target/arm/gdbstub.c   | 34 ++
 target/hexagon/cpu.c   |  3 +--
 target/loongarch/gdbstub.c |  2 +-
 target/m68k/helper.c   |  6 +++---
 target/microblaze/cpu.c|  5 +++--
 target/ppc/gdbstub.c   | 11 ++-
 target/riscv/gdbstub.c | 18 ++
 target/s390x/gdbstub.c | 28 +++-
 10 files changed, 57 insertions(+), 65 deletions(-)

diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index d9ef2ccbff..5525eea090 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -37,7 +37,7 @@ typedef int (*gdb_set_reg_cb)(CPUArchState *env, uint8_t 
*buf, int reg);
  */
 void gdb_register_coprocessor(CPUState *cpu,
   gdb_get_reg_cb get_reg, gdb_set_reg_cb set_reg,
-  int num_regs, const char *xml, int g_pos);
+  const GDBFeature *feature, int g_pos);
 
 /**
  * gdbserver_start: start the gdb server
diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index 258dd54c90..20586e0b6a 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -532,7 +532,7 @@ static int gdb_write_register(CPUState *cpu, uint8_t 
*mem_buf, int reg)
 
 void gdb_register_coprocessor(CPUState *cpu,
   gdb_get_reg_cb get_reg, gdb_set_reg_cb set_reg,
-  int num_regs, const char *xml, int g_pos)
+  const GDBFeature *feature, int g_pos)
 {
 GDBRegisterState *s;
 guint i;
@@ -541,7 +541,7 @@ void gdb_register_coprocessor(CPUState *cpu,
 for (i = 0; i < cpu->gdb_regs->len; i++) {
 /* Check for duplicates.  */
 s = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
-if (strcmp(s->xml, xml) == 0) {
+if (strcmp(s->xml, feature->xmlname) == 0) {
 return;
 }
 }
@@ -553,17 +553,18 @@ void gdb_register_coprocessor(CPUState *cpu,
 g_array_set_size(cpu->gdb_regs, i + 1);
 s = &g_array_index(cpu->gdb_regs, GDBRegisterState, i);
 s->base_reg = cpu->gdb_num_regs;
-s->num_regs = num_regs;
+s->num_regs = feature->num_regs;
 s->get_reg = get_reg;
 s->set_reg = set_reg;
-s->xml = xml;
+s->xml = feature->xml;
 
 /* Add to end of list.  */
-cpu->gdb_num_regs += num_regs;
+cpu->gdb_num_regs += feature->num_regs;
 if (g_pos) {
 if (g_pos != s->base_reg) {
 error_report("Error: Bad gdb register numbering for '%s', "
- "expected %d got %d", xml, g_pos, s->base_reg);
+ "expected %d got %d", feature->xml,
+ g_pos, s->base_reg);
 } else {
 cpu->gdb_num_g_regs = cpu->gdb_num_regs;
 }
diff --git a/target/arm/gdbstub.c b/target/arm/gdbstub.c
index fc0aa6f235..99040e0c4c 100644
--- a/target/arm/gdbstub.c
+++ b/target/arm/gdbstub.c
@@ -472,14 +472,14 @@ void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
  */
 #ifdef TARGET_AARCH64
 if (isar_feature_aa64_sve(&cpu->isar)) {
-int nreg = arm_gen_dynamic_svereg_feature(cs)->num_regs;
+GDBFeature *feature = arm_gen_dynamic_svereg_feature(cs);
 gdb_register_coprocessor(cs, aarch64_gdb_get_sve_reg,
- aarch64_gdb_set_sve_reg, nreg,
- "sve-registers.xml", 0);
+ aarch64_gdb_set_sve_reg, feature, 0);
 } else {
 gdb_register_coprocessor(cs, aarch64_gdb_get_fpu_reg,
  aarch64_gdb_set_fpu_reg,
- 34, "aarch64-fpu.xml", 0);
+ 
gdb_find_static_feature("aarch64-fpu.xml"),
+ 0);
 }
 /*
  * Note that we report pauth information via the feature name
@@ -490,19 +490,22 @@ void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
 if (isar_feature_aa64_pauth(&cpu->isar)) {
 gdb_register_coprocessor(cs, aarch64_gdb_get_pauth_reg,
  aarch64_gdb_set_pauth_reg,
- 4, "aarch64-pauth.xml", 0);
+ 
gdb_find_static_feature("aarch64-pauth.xml"),
+ 0);
 }
 #endif
 } else {
 if (arm_feature(env, ARM_FEATURE_NEON)) {
 gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
-   

[PATCH v9 09/23] target/riscv: Use GDBFeature for dynamic XML

[Akihiko Odaki]

In preparation for a change to use GDBFeature as a parameter of
gdb_register_coprocessor(), convert the internal representation of
dynamic feature from plain XML to GDBFeature.

Signed-off-by: Akihiko Odaki 
---
 target/riscv/cpu.h |  5 +--
 target/riscv/cpu.c |  4 +--
 target/riscv/gdbstub.c | 77 ++
 3 files changed, 38 insertions(+), 48 deletions(-)

diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h
index 9f9cb6cd2a..85f978f7d4 100644
--- a/target/riscv/cpu.h
+++ b/target/riscv/cpu.h
@@ -23,6 +23,7 @@
 #include "hw/core/cpu.h"
 #include "hw/registerfields.h"
 #include "exec/cpu-defs.h"
+#include "exec/gdbstub.h"
 #include "qemu/cpu-float.h"
 #include "qom/object.h"
 #include "qemu/int128.h"
@@ -389,8 +390,8 @@ struct ArchCPU {
 
 CPURISCVState env;
 
-char *dyn_csr_xml;
-char *dyn_vreg_xml;
+GDBFeature dyn_csr_feature;
+GDBFeature dyn_vreg_feature;
 
 /* Configuration Settings */
 RISCVCPUConfig cfg;
diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c
index 0c087df9f9..ce31308366 100644
--- a/target/riscv/cpu.c
+++ b/target/riscv/cpu.c
@@ -2066,9 +2066,9 @@ static const char *riscv_gdb_get_dynamic_xml(CPUState 
*cs, const char *xmlname)
 RISCVCPU *cpu = RISCV_CPU(cs);
 
 if (strcmp(xmlname, "riscv-csr.xml") == 0) {
-return cpu->dyn_csr_xml;
+return cpu->dyn_csr_feature.xml;
 } else if (strcmp(xmlname, "riscv-vector.xml") == 0) {
-return cpu->dyn_vreg_xml;
+return cpu->dyn_vreg_feature.xml;
 }
 
 return NULL;
diff --git a/target/riscv/gdbstub.c b/target/riscv/gdbstub.c
index b9528cef5b..b7159f1db8 100644
--- a/target/riscv/gdbstub.c
+++ b/target/riscv/gdbstub.c
@@ -214,13 +214,14 @@ static int riscv_gdb_set_virtual(CPURISCVState *cs, 
uint8_t *mem_buf, int n)
 return 0;
 }
 
-static int riscv_gen_dynamic_csr_xml(CPUState *cs, int base_reg)
+static GDBFeature *riscv_gen_dynamic_csr_feature(CPUState *cs)
 {
 RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cs);
 RISCVCPU *cpu = RISCV_CPU(cs);
 CPURISCVState *env = &cpu->env;
-GString *s = g_string_new(NULL);
+GDBFeatureBuilder builder;
 riscv_csr_predicate_fn predicate;
+const char *name;
 int bitsize = 16 << mcc->misa_mxl_max;
 int i;
 
@@ -233,9 +234,8 @@ static int riscv_gen_dynamic_csr_xml(CPUState *cs, int 
base_reg)
 bitsize = 64;
 }
 
-g_string_printf(s, "");
-g_string_append_printf(s, "");
-g_string_append_printf(s, "");
+gdb_feature_builder_init(&builder, &cpu->dyn_csr_feature,
+ "org.gnu.gdb.riscv.csr", "riscv-csr.xml");
 
 for (i = 0; i < CSR_TABLE_SIZE; i++) {
 if (env->priv_ver < csr_ops[i].min_priv_ver) {
@@ -243,72 +243,63 @@ static int riscv_gen_dynamic_csr_xml(CPUState *cs, int 
base_reg)
 }
 predicate = csr_ops[i].predicate;
 if (predicate && (predicate(env, i) == RISCV_EXCP_NONE)) {
-if (csr_ops[i].name) {
-g_string_append_printf(s, "", base_reg + i);
+
+gdb_feature_builder_append_reg(&builder, name, bitsize,
+   "int", NULL);
 }
 }
 
-g_string_append_printf(s, "");
-
-cpu->dyn_csr_xml = g_string_free(s, false);
+gdb_feature_builder_end(&builder);
 
 #if !defined(CONFIG_USER_ONLY)
 env->debugger = false;
 #endif
 
-return CSR_TABLE_SIZE;
+return &cpu->dyn_csr_feature;
 }
 
-static int ricsv_gen_dynamic_vector_xml(CPUState *cs, int base_reg)
+static GDBFeature *ricsv_gen_dynamic_vector_feature(CPUState *cs)
 {
 RISCVCPU *cpu = RISCV_CPU(cs);
-GString *s = g_string_new(NULL);
-g_autoptr(GString) ts = g_string_new("");
+GDBFeatureBuilder builder;
 int reg_width = cpu->cfg.vlen;
-int num_regs = 0;
 int i;
 
-g_string_printf(s, "");
-g_string_append_printf(s, "");
-g_string_append_printf(s, "");
+gdb_feature_builder_init(&builder, &cpu->dyn_vreg_feature,
+ "org.gnu.gdb.riscv.vector", "riscv-vector.xml");
 
 /* First define types and totals in a whole VL */
 for (i = 0; i < ARRAY_SIZE(vec_lanes); i++) {
 int count = reg_width / vec_lanes[i].size;
-g_string_printf(ts, "%s", vec_lanes[i].id);
-g_string_append_printf(s,
-   "",
-   ts->str, vec_lanes[i].gdb_type, count);
+gdb_feature_builder_append_tag(
+&builder, "",
+vec_lanes[i].id, vec_lanes[i].gdb_type, count);
 }
 
 /* Define unions */
-g_string_append_printf(s, "");
+gdb_feature_builder_append_tag(&builder, "");
 for (i = 0; i < ARRAY_SIZE(vec_lanes); i++) {
-g_string_append_printf(s, "",
-   vec_lanes[i].suffix,
-   vec_lanes[i].id);
+gdb_feature_builder_append_tag(&builder,
+   "",
+   

[PATCH v9 18/23] cpu: Call plugin hooks only when ready

[Akihiko Odaki]

The initialization and exit hooks will not affect the state of vCPU
outside TCG context, but they may depend on the state of vCPU.
Therefore, it's better to call plugin hooks after the vCPU state is
fully initialized and before it gets uninitialized.

Signed-off-by: Akihiko Odaki 
---
 cpu-target.c | 11 ---
 hw/core/cpu-common.c | 10 ++
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/cpu-target.c b/cpu-target.c
index 658d179582..7acb65fd39 100644
--- a/cpu-target.c
+++ b/cpu-target.c
@@ -42,7 +42,6 @@
 #include "hw/core/accel-cpu.h"
 #include "trace/trace-root.h"
 #include "qemu/accel.h"
-#include "qemu/plugin.h"
 
 uintptr_t qemu_host_page_size;
 intptr_t qemu_host_page_mask;
@@ -143,11 +142,6 @@ void cpu_exec_realizefn(CPUState *cpu, Error **errp)
 /* Wait until cpu initialization complete before exposing cpu. */
 cpu_list_add(cpu);
 
-/* Plugin initialization must wait until cpu_index assigned. */
-if (tcg_enabled()) {
-qemu_plugin_vcpu_init_hook(cpu);
-}
-
 #ifdef CONFIG_USER_ONLY
 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL ||
qdev_get_vmsd(DEVICE(cpu))->unmigratable);
@@ -174,11 +168,6 @@ void cpu_exec_unrealizefn(CPUState *cpu)
 }
 #endif
 
-/* Call the plugin hook before clearing cpu->cpu_index in cpu_list_remove 
*/
-if (tcg_enabled()) {
-qemu_plugin_vcpu_exit_hook(cpu);
-}
-
 cpu_list_remove(cpu);
 /*
  * Now that the vCPU has been removed from the RCU list, we can call
diff --git a/hw/core/cpu-common.c b/hw/core/cpu-common.c
index 0767714048..9805d2c301 100644
--- a/hw/core/cpu-common.c
+++ b/hw/core/cpu-common.c
@@ -210,6 +210,11 @@ static void cpu_common_realizefn(DeviceState *dev, Error 
**errp)
 cpu_resume(cpu);
 }
 
+/* Plugin initialization must wait until the cpu is fully realized. */
+if (tcg_enabled()) {
+qemu_plugin_vcpu_init_hook(cpu);
+}
+
 /* NOTE: latest generic point where the cpu is fully realized */
 }
 
@@ -217,6 +222,11 @@ static void cpu_common_unrealizefn(DeviceState *dev)
 {
 CPUState *cpu = CPU(dev);
 
+/* Call the plugin hook before clearing the cpu is fully unrealized */
+if (tcg_enabled()) {
+qemu_plugin_vcpu_exit_hook(cpu);
+}
+
 /* NOTE: latest generic point before the cpu is fully unrealized */
 cpu_exec_unrealizefn(cpu);
 }
-- 
2.42.0

[PATCH v9 06/23] gdbstub: Introduce GDBFeatureBuilder

[Akihiko Odaki]

GDBFeatureBuilder unifies the logic to generate dynamic GDBFeature.

Signed-off-by: Akihiko Odaki 
Reviewed-by: Richard Henderson 
---
 include/exec/gdbstub.h | 20 ++
 gdbstub/gdbstub.c  | 59 ++
 2 files changed, 79 insertions(+)

diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index 071021415a..d9ef2ccbff 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -16,6 +16,11 @@ typedef struct GDBFeature {
 int num_regs;
 } GDBFeature;
 
+typedef struct GDBFeatureBuilder {
+GDBFeature *feature;
+GPtrArray *xml;
+} GDBFeatureBuilder;
+
 
 /* Get or set a register.  Returns the size of the register.  */
 typedef int (*gdb_get_reg_cb)(CPUArchState *env, GByteArray *buf, int reg);
@@ -44,6 +49,21 @@ void gdb_register_coprocessor(CPUState *cpu,
  */
 int gdbserver_start(const char *port_or_device);
 
+void gdb_feature_builder_init(GDBFeatureBuilder *builder, GDBFeature *feature,
+  const char *name, const char *xmlname);
+
+void gdb_feature_builder_append_tag(const GDBFeatureBuilder *builder,
+const char *format, ...)
+G_GNUC_PRINTF(2, 3);
+
+int gdb_feature_builder_append_reg(const GDBFeatureBuilder *builder,
+   const char *name,
+   int bitsize,
+   const char *type,
+   const char *group);
+
+void gdb_feature_builder_end(const GDBFeatureBuilder *builder);
+
 const GDBFeature *gdb_find_static_feature(const char *xmlname);
 
 void gdb_set_stop_cpu(CPUState *cpu);
diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index 44c9167542..258dd54c90 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -420,6 +420,65 @@ static const char *get_feature_xml(const char *p, const 
char **newp,
 return NULL;
 }
 
+void gdb_feature_builder_init(GDBFeatureBuilder *builder, GDBFeature *feature,
+  const char *name, const char *xmlname)
+{
+char *header = g_markup_printf_escaped(
+""
+""
+"",
+name);
+
+builder->feature = feature;
+builder->xml = g_ptr_array_new();
+g_ptr_array_add(builder->xml, header);
+feature->xmlname = xmlname;
+feature->num_regs = 0;
+}
+
+void gdb_feature_builder_append_tag(const GDBFeatureBuilder *builder,
+const char *format, ...)
+{
+va_list ap;
+va_start(ap, format);
+g_ptr_array_add(builder->xml, g_markup_vprintf_escaped(format, ap));
+va_end(ap);
+}
+
+int gdb_feature_builder_append_reg(const GDBFeatureBuilder *builder,
+   const char *name,
+   int bitsize,
+   const char *type,
+   const char *group)
+{
+if (group) {
+gdb_feature_builder_append_tag(
+builder,
+"",
+name, bitsize, type, group);
+} else {
+gdb_feature_builder_append_tag(
+builder, "",
+name, bitsize, type);
+}
+
+return builder->feature->num_regs++;
+}
+
+void gdb_feature_builder_end(const GDBFeatureBuilder *builder)
+{
+g_ptr_array_add(builder->xml, (void *)"");
+g_ptr_array_add(builder->xml, NULL);
+
+builder->feature->xml = g_strjoinv(NULL, (void *)builder->xml->pdata);
+
+for (guint i = 0; i < builder->xml->len - 2; i++) {
+g_free(g_ptr_array_index(builder->xml, i));
+}
+
+g_ptr_array_free(builder->xml, TRUE);
+}
+
 const GDBFeature *gdb_find_static_feature(const char *xmlname)
 {
 const GDBFeature *feature;
-- 
2.42.0

[PATCH v9 08/23] target/ppc: Use GDBFeature for dynamic XML

[Akihiko Odaki]

In preparation for a change to use GDBFeature as a parameter of
gdb_register_coprocessor(), convert the internal representation of
dynamic feature from plain XML to GDBFeature.

Signed-off-by: Akihiko Odaki 
Reviewed-by: Richard Henderson 
---
 target/ppc/cpu-qom.h  |  4 ++--
 target/ppc/cpu.h  |  2 +-
 target/ppc/cpu_init.c |  2 +-
 target/ppc/gdbstub.c  | 45 ++-
 4 files changed, 18 insertions(+), 35 deletions(-)

diff --git a/target/ppc/cpu-qom.h b/target/ppc/cpu-qom.h
index be33786bd8..8d5ebba5d3 100644
--- a/target/ppc/cpu-qom.h
+++ b/target/ppc/cpu-qom.h
@@ -20,6 +20,7 @@
 #ifndef QEMU_PPC_CPU_QOM_H
 #define QEMU_PPC_CPU_QOM_H
 
+#include "exec/gdbstub.h"
 #include "hw/core/cpu.h"
 #include "qom/object.h"
 
@@ -186,8 +187,7 @@ struct PowerPCCPUClass {
 int bfd_mach;
 uint32_t l1_dcache_size, l1_icache_size;
 #ifndef CONFIG_USER_ONLY
-unsigned int gdb_num_sprs;
-const char *gdb_spr_xml;
+GDBFeature gdb_spr;
 #endif
 const PPCHash64Options *hash64_opts;
 struct ppc_radix_page_info *radix_page_info;
diff --git a/target/ppc/cpu.h b/target/ppc/cpu.h
index 30392ebeee..84cdc3055f 100644
--- a/target/ppc/cpu.h
+++ b/target/ppc/cpu.h
@@ -1384,7 +1384,7 @@ int ppc_cpu_gdb_write_register(CPUState *cpu, uint8_t 
*buf, int reg);
 int ppc_cpu_gdb_write_register_apple(CPUState *cpu, uint8_t *buf, int reg);
 #ifndef CONFIG_USER_ONLY
 hwaddr ppc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-void ppc_gdb_gen_spr_xml(PowerPCCPU *cpu);
+void ppc_gdb_gen_spr_feature(PowerPCCPU *cpu);
 const char *ppc_gdb_get_dynamic_xml(CPUState *cs, const char *xml_name);
 #endif
 int ppc64_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c
index 40fe14a6c2..af43e73af2 100644
--- a/target/ppc/cpu_init.c
+++ b/target/ppc/cpu_init.c
@@ -6683,7 +6683,7 @@ static void init_ppc_proc(PowerPCCPU *cpu)
 (*pcc->init_proc)(env);
 
 #if !defined(CONFIG_USER_ONLY)
-ppc_gdb_gen_spr_xml(cpu);
+ppc_gdb_gen_spr_feature(cpu);
 #endif
 
 /* MSR bits & flags consistency checks */
diff --git a/target/ppc/gdbstub.c b/target/ppc/gdbstub.c
index ec5731e5d6..43de40ac36 100644
--- a/target/ppc/gdbstub.c
+++ b/target/ppc/gdbstub.c
@@ -300,15 +300,21 @@ int ppc_cpu_gdb_write_register_apple(CPUState *cs, 
uint8_t *mem_buf, int n)
 }
 
 #ifndef CONFIG_USER_ONLY
-void ppc_gdb_gen_spr_xml(PowerPCCPU *cpu)
+void ppc_gdb_gen_spr_feature(PowerPCCPU *cpu)
 {
 PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
 CPUPPCState *env = &cpu->env;
-GString *xml;
-char *spr_name;
+GDBFeatureBuilder builder;
 unsigned int num_regs = 0;
 int i;
 
+if (pcc->gdb_spr.xml) {
+return;
+}
+
+gdb_feature_builder_init(&builder, &pcc->gdb_spr,
+ "org.qemu.power.spr", "power-spr.xml");
+
 for (i = 0; i < ARRAY_SIZE(env->spr_cb); i++) {
 ppc_spr_t *spr = &env->spr_cb[i];
 
@@ -326,35 +332,12 @@ void ppc_gdb_gen_spr_xml(PowerPCCPU *cpu)
  */
 spr->gdb_id = num_regs;
 num_regs++;
-}
-
-if (pcc->gdb_spr_xml) {
-return;
-}
-
-xml = g_string_new("");
-g_string_append(xml, "");
-g_string_append(xml, "");
 
-for (i = 0; i < ARRAY_SIZE(env->spr_cb); i++) {
-ppc_spr_t *spr = &env->spr_cb[i];
-
-if (!spr->name) {
-continue;
-}
-
-spr_name = g_ascii_strdown(spr->name, -1);
-g_string_append_printf(xml, "");
+gdb_feature_builder_append_reg(&builder, g_ascii_strdown(spr->name, 
-1),
+   TARGET_LONG_BITS, "int", "spr");
 }
 
-g_string_append(xml, "");
-
-pcc->gdb_num_sprs = num_regs;
-pcc->gdb_spr_xml = g_string_free(xml, false);
+gdb_feature_builder_end(&builder);
 }
 
 const char *ppc_gdb_get_dynamic_xml(CPUState *cs, const char *xml_name)
@@ -362,7 +345,7 @@ const char *ppc_gdb_get_dynamic_xml(CPUState *cs, const 
char *xml_name)
 PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cs);
 
 if (strcmp(xml_name, "power-spr.xml") == 0) {
-return pcc->gdb_spr_xml;
+return pcc->gdb_spr.xml;
 }
 return NULL;
 }
@@ -600,6 +583,6 @@ void ppc_gdb_init(CPUState *cs, PowerPCCPUClass *pcc)
 }
 #ifndef CONFIG_USER_ONLY
 gdb_register_coprocessor(cs, gdb_get_spr_reg, gdb_set_spr_reg,
- pcc->gdb_num_sprs, "power-spr.xml", 0);
+ pcc->gdb_spr.num_regs, "power-spr.xml", 0);
 #endif
 }
-- 
2.42.0

[PATCH v9 20/23] plugins: Use different helpers when reading registers

[Akihiko Odaki]

This avoids optimizations incompatible when reading registers.

Signed-off-by: Akihiko Odaki 
---
 accel/tcg/plugin-helpers.h |  3 ++-
 include/exec/plugin-gen.h  |  4 ++--
 include/hw/core/cpu.h  |  4 ++--
 include/qemu/plugin.h  |  3 +++
 plugins/plugin.h   |  5 +++--
 accel/tcg/plugin-gen.c | 41 --
 accel/tcg/translator.c |  2 +-
 plugins/api.c  | 10 --
 plugins/core.c | 28 --
 9 files changed, 68 insertions(+), 32 deletions(-)

diff --git a/accel/tcg/plugin-helpers.h b/accel/tcg/plugin-helpers.h
index 8e685e0654..11796436f3 100644
--- a/accel/tcg/plugin-helpers.h
+++ b/accel/tcg/plugin-helpers.h
@@ -1,4 +1,5 @@
 #ifdef CONFIG_PLUGIN
-DEF_HELPER_FLAGS_2(plugin_vcpu_udata_cb, TCG_CALL_NO_RWG | TCG_CALL_PLUGIN, 
void, i32, ptr)
+DEF_HELPER_FLAGS_2(plugin_vcpu_udata_cb_no_wg, TCG_CALL_NO_WG | 
TCG_CALL_PLUGIN, void, i32, ptr)
+DEF_HELPER_FLAGS_2(plugin_vcpu_udata_cb_no_rwg, TCG_CALL_NO_RWG | 
TCG_CALL_PLUGIN, void, i32, ptr)
 DEF_HELPER_FLAGS_4(plugin_vcpu_mem_cb, TCG_CALL_NO_RWG | TCG_CALL_PLUGIN, 
void, i32, i32, i64, ptr)
 #endif
diff --git a/include/exec/plugin-gen.h b/include/exec/plugin-gen.h
index c4552b5061..b964e1eb5c 100644
--- a/include/exec/plugin-gen.h
+++ b/include/exec/plugin-gen.h
@@ -22,7 +22,7 @@ bool plugin_gen_tb_start(CPUState *cpu, const struct 
DisasContextBase *db,
  bool supress);
 void plugin_gen_tb_end(CPUState *cpu, size_t num_insns);
 void plugin_gen_insn_start(CPUState *cpu, const struct DisasContextBase *db);
-void plugin_gen_insn_end(void);
+void plugin_gen_insn_end(CPUState *cpu);
 
 void plugin_gen_disable_mem_helpers(void);
 void plugin_gen_empty_mem_callback(TCGv_i64 addr, uint32_t info);
@@ -39,7 +39,7 @@ static inline
 void plugin_gen_insn_start(CPUState *cpu, const struct DisasContextBase *db)
 { }
 
-static inline void plugin_gen_insn_end(void)
+static inline void plugin_gen_insn_end(CPUState *cpu)
 { }
 
 static inline void plugin_gen_tb_end(CPUState *cpu, size_t num_insns)
diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h
index d2e70643f2..dbdca8b105 100644
--- a/include/hw/core/cpu.h
+++ b/include/hw/core/cpu.h
@@ -437,7 +437,7 @@ struct qemu_work_item;
  * @trace_dstate_delayed: Delayed changes to trace_dstate (includes all changes
  *to @trace_dstate).
  * @trace_dstate: Dynamic tracing state of events for this vCPU (bitmask).
- * @plugin_mask: Plugin event bitmap. Modified only via async work.
+ * @plugin_flags: Plugin flags. Modified only via async work.
  * @ignore_memory_transaction_failures: Cached copy of the MachineState
  *flag of the same name: allows the board to suppress calling of the
  *CPU do_transaction_failed hook function.
@@ -529,7 +529,7 @@ struct CPUState {
 /* Use by accel-block: CPU is executing an ioctl() */
 QemuLockCnt in_ioctl_lock;
 
-DECLARE_BITMAP(plugin_mask, QEMU_PLUGIN_EV_MAX);
+unsigned long plugin_flags;
 
 #ifdef CONFIG_PLUGIN
 GArray *plugin_mem_cbs;
diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h
index bc0781cab8..ba574ddc11 100644
--- a/include/qemu/plugin.h
+++ b/include/qemu/plugin.h
@@ -16,6 +16,9 @@
 #include "exec/memopidx.h"
 #include "hw/core/cpu.h"
 
+#define QEMU_PLUGIN_FLAG_TB_CB_READ QEMU_PLUGIN_EV_MAX
+#define QEMU_PLUGIN_FLAG_INSN_CB_READ (QEMU_PLUGIN_EV_MAX + 1)
+
 /*
  * Option parsing/processing.
  * Note that we can load an arbitrary number of plugins.
diff --git a/plugins/plugin.h b/plugins/plugin.h
index 5eb2fdbc85..ba0417194f 100644
--- a/plugins/plugin.h
+++ b/plugins/plugin.h
@@ -16,6 +16,7 @@
 #include "qemu/qht.h"
 
 #define QEMU_PLUGIN_MIN_VERSION 0
+#define QEMU_PLUGIN_FLAG_INSIN_CB_READ QEMU_PLUGIN_EV_MAX
 
 /* global state */
 struct qemu_plugin_state {
@@ -31,7 +32,7 @@ struct qemu_plugin_state {
  * but with the HT we avoid adding a field to CPUState.
  */
 GHashTable *cpu_ht;
-DECLARE_BITMAP(mask, QEMU_PLUGIN_EV_MAX);
+unsigned long flags;
 /*
  * @lock protects the struct as well as ctx->uninstalling.
  * The lock must be acquired by all API ops.
@@ -86,7 +87,7 @@ plugin_register_cb_udata(qemu_plugin_id_t id, enum 
qemu_plugin_event ev,
 void
 plugin_register_dyn_cb__udata(GArray **arr,
   qemu_plugin_vcpu_udata_cb_t cb,
-  enum qemu_plugin_cb_flags flags, void *udata);
+  unsigned int flags, void *udata);
 
 
 void plugin_register_vcpu_mem_cb(GArray **arr,
diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c
index 78b331b251..3bddd4d3c5 100644
--- a/accel/tcg/plugin-gen.c
+++ b/accel/tcg/plugin-gen.c
@@ -90,7 +90,10 @@ enum plugin_gen_cb {
  * These helpers are stubs that get dynamically switched out for calls
  * direct to the plugin if they are subscribed to.
  */
-void HELPER(plugin_vcpu_udata_cb)(uint32_t cpu_index, void *udata)
+void HELPER(plugin_vcpu_udata

[PATCH v9 15/23] hw/core/cpu: Remove gdb_get_dynamic_xml member

[Akihiko Odaki]

This function is no longer used.

Signed-off-by: Akihiko Odaki 
---
 include/hw/core/cpu.h |  4 
 target/arm/cpu.h  |  6 --
 target/ppc/cpu.h  |  1 -
 target/arm/cpu.c  |  1 -
 target/arm/gdbstub.c  | 18 --
 target/ppc/cpu_init.c |  3 ---
 target/ppc/gdbstub.c  | 10 --
 target/riscv/cpu.c| 14 --
 8 files changed, 57 deletions(-)

diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h
index 11d4b5cd0c..d2e70643f2 100644
--- a/include/hw/core/cpu.h
+++ b/include/hw/core/cpu.h
@@ -134,9 +134,6 @@ struct SysemuCPUOps;
  *   before the insn which triggers a watchpoint rather than after it.
  * @gdb_arch_name: Optional callback that returns the architecture name known
  * to GDB. The caller must free the returned string with g_free.
- * @gdb_get_dynamic_xml: Callback to return dynamically generated XML for the
- *   gdb stub. Returns a pointer to the XML contents for the specified XML file
- *   or NULL if the CPU doesn't have a dynamically generated content for it.
  * @disas_set_info: Setup architecture specific components of disassembly info
  * @adjust_watchpoint_address: Perform a target-specific adjustment to an
  * address before attempting to match it against watchpoints.
@@ -167,7 +164,6 @@ struct CPUClass {
 
 const char *gdb_core_xml_file;
 const gchar * (*gdb_arch_name)(CPUState *cpu);
-const char * (*gdb_get_dynamic_xml)(CPUState *cpu, const char *xmlname);
 
 void (*disas_set_info)(CPUState *cpu, disassemble_info *info);
 
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index 3702ddaab8..0e4f1d7d0c 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -1137,12 +1137,6 @@ hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cpu, 
vaddr addr,
 int arm_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg);
 int arm_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 
-/* Returns the dynamically generated XML for the gdb stub.
- * Returns a pointer to the XML contents for the specified XML file or NULL
- * if the XML name doesn't match the predefined one.
- */
-const char *arm_gdb_get_dynamic_xml(CPUState *cpu, const char *xmlname);
-
 int arm_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
  int cpuid, DumpState *s);
 int arm_cpu_write_elf32_note(WriteCoreDumpFunction f, CPUState *cs,
diff --git a/target/ppc/cpu.h b/target/ppc/cpu.h
index 84cdc3055f..1782b21edb 100644
--- a/target/ppc/cpu.h
+++ b/target/ppc/cpu.h
@@ -1385,7 +1385,6 @@ int ppc_cpu_gdb_write_register_apple(CPUState *cpu, 
uint8_t *buf, int reg);
 #ifndef CONFIG_USER_ONLY
 hwaddr ppc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 void ppc_gdb_gen_spr_feature(PowerPCCPU *cpu);
-const char *ppc_gdb_get_dynamic_xml(CPUState *cs, const char *xml_name);
 #endif
 int ppc64_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
int cpuid, DumpState *s);
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 6d9bf6a14e..e7fe11339b 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -2392,7 +2392,6 @@ static void arm_cpu_class_init(ObjectClass *oc, void 
*data)
 cc->sysemu_ops = &arm_sysemu_ops;
 #endif
 cc->gdb_arch_name = arm_gdb_arch_name;
-cc->gdb_get_dynamic_xml = arm_gdb_get_dynamic_xml;
 cc->gdb_stop_before_watchpoint = true;
 cc->disas_set_info = arm_disas_set_info;
 
diff --git a/target/arm/gdbstub.c b/target/arm/gdbstub.c
index aa2c6c1bde..f9774b5547 100644
--- a/target/arm/gdbstub.c
+++ b/target/arm/gdbstub.c
@@ -463,24 +463,6 @@ static GDBFeature 
*arm_gen_dynamic_m_secextreg_feature(CPUState *cs)
 #endif
 #endif /* CONFIG_TCG */
 
-const char *arm_gdb_get_dynamic_xml(CPUState *cs, const char *xmlname)
-{
-ARMCPU *cpu = ARM_CPU(cs);
-
-if (strcmp(xmlname, "system-registers.xml") == 0) {
-return cpu->dyn_sysreg_feature.desc.xml;
-} else if (strcmp(xmlname, "sve-registers.xml") == 0) {
-return cpu->dyn_svereg_feature.desc.xml;
-} else if (strcmp(xmlname, "arm-m-system.xml") == 0) {
-return cpu->dyn_m_systemreg_feature.desc.xml;
-#ifndef CONFIG_USER_ONLY
-} else if (strcmp(xmlname, "arm-m-secext.xml") == 0) {
-return cpu->dyn_m_secextreg_feature.desc.xml;
-#endif
-}
-return NULL;
-}
-
 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
 {
 CPUState *cs = CPU(cpu);
diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c
index af43e73af2..d8a992a45d 100644
--- a/target/ppc/cpu_init.c
+++ b/target/ppc/cpu_init.c
@@ -7384,9 +7384,6 @@ static void ppc_cpu_class_init(ObjectClass *oc, void 
*data)
 #endif
 
 cc->gdb_num_core_regs = 71;
-#ifndef CONFIG_USER_ONLY
-cc->gdb_get_dynamic_xml = ppc_gdb_get_dynamic_xml;
-#endif
 #ifdef USE_APPLE_GDB
 cc->gdb_read_register = ppc_cpu_gdb_read_register_apple;
 cc->gdb_write_register = ppc_cpu_gdb_write_register_apple;
diff --git a/target/ppc/gdbstub.c b/target/ppc/gdbstub.c
index 2392625441..a20d5922dd 100

[PATCH v9 02/23] target/riscv: Remove misa_mxl validation

[Akihiko Odaki]

It is initialized with a simple assignment and there is little room for
error. In fact, the validation is even more complex.

Signed-off-by: Akihiko Odaki 
---
 target/riscv/cpu.c | 14 ++
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c
index 3bb1ce90f9..1306054d27 100644
--- a/target/riscv/cpu.c
+++ b/target/riscv/cpu.c
@@ -1087,11 +1087,10 @@ static void 
riscv_cpu_disable_priv_spec_isa_exts(RISCVCPU *cpu)
 }
 }
 
-static void riscv_cpu_validate_misa_mxl(RISCVCPU *cpu, Error **errp)
+static void riscv_cpu_validate_misa_mxl(RISCVCPU *cpu)
 {
 RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cpu);
 CPUClass *cc = CPU_CLASS(mcc);
-CPURISCVState *env = &cpu->env;
 
 /* Validate that MISA_MXL is set properly. */
 switch (mcc->misa_mxl_max) {
@@ -1107,11 +1106,6 @@ static void riscv_cpu_validate_misa_mxl(RISCVCPU *cpu, 
Error **errp)
 default:
 g_assert_not_reached();
 }
-
-if (mcc->misa_mxl_max != env->misa_mxl) {
-error_setg(errp, "misa_mxl_max must be equal to misa_mxl");
-return;
-}
 }
 
 /*
@@ -1493,11 +1487,7 @@ static void riscv_cpu_realize_tcg(DeviceState *dev, 
Error **errp)
 return;
 }
 
-riscv_cpu_validate_misa_mxl(cpu, &local_err);
-if (local_err != NULL) {
-error_propagate(errp, local_err);
-return;
-}
+riscv_cpu_validate_misa_mxl(cpu);
 
 riscv_cpu_validate_priv_spec(cpu, &local_err);
 if (local_err != NULL) {
-- 
2.42.0

[PATCH v9 05/23] gdbstub: Introduce gdb_find_static_feature()

[Akihiko Odaki]

This function is useful to determine the number of registers exposed to
GDB from the XML name.

Signed-off-by: Akihiko Odaki 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Alex Bennée 
Reviewed-by: Richard Henderson 
---
 include/exec/gdbstub.h |  2 ++
 gdbstub/gdbstub.c  | 13 +
 2 files changed, 15 insertions(+)

diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index a43aa34dad..071021415a 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -44,6 +44,8 @@ void gdb_register_coprocessor(CPUState *cpu,
  */
 int gdbserver_start(const char *port_or_device);
 
+const GDBFeature *gdb_find_static_feature(const char *xmlname);
+
 void gdb_set_stop_cpu(CPUState *cpu);
 
 /* in gdbstub-xml.c, generated by scripts/feature_to_c.py */
diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index b1532118d1..44c9167542 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -420,6 +420,19 @@ static const char *get_feature_xml(const char *p, const 
char **newp,
 return NULL;
 }
 
+const GDBFeature *gdb_find_static_feature(const char *xmlname)
+{
+const GDBFeature *feature;
+
+for (feature = gdb_static_features; feature->xmlname; feature++) {
+if (!strcmp(feature->xmlname, xmlname)) {
+return feature;
+}
+}
+
+g_assert_not_reached();
+}
+
 static int gdb_read_register(CPUState *cpu, GByteArray *buf, int reg)
 {
 CPUClass *cc = CPU_GET_CLASS(cpu);
-- 
2.42.0

[PATCH v9 14/23] gdbstub: Infer number of core registers from XML

[Akihiko Odaki]

GDBFeature has the num_regs member so use it where applicable to
remove magic numbers.

Signed-off-by: Akihiko Odaki 
---
 include/hw/core/cpu.h   | 3 ++-
 target/s390x/cpu.h  | 2 --
 gdbstub/gdbstub.c   | 5 -
 target/arm/cpu.c| 1 -
 target/arm/cpu64.c  | 1 -
 target/avr/cpu.c| 1 -
 target/hexagon/cpu.c| 1 -
 target/i386/cpu.c   | 2 --
 target/loongarch/cpu.c  | 2 --
 target/m68k/cpu.c   | 1 -
 target/microblaze/cpu.c | 1 -
 target/riscv/cpu.c  | 1 -
 target/rx/cpu.c | 1 -
 target/s390x/cpu.c  | 1 -
 14 files changed, 6 insertions(+), 17 deletions(-)

diff --git a/include/hw/core/cpu.h b/include/hw/core/cpu.h
index 3968369554..11d4b5cd0c 100644
--- a/include/hw/core/cpu.h
+++ b/include/hw/core/cpu.h
@@ -127,7 +127,8 @@ struct SysemuCPUOps;
  * @gdb_adjust_breakpoint: Callback for adjusting the address of a
  *   breakpoint.  Used by AVR to handle a gdb mis-feature with
  *   its Harvard architecture split code and data.
- * @gdb_num_core_regs: Number of core registers accessible to GDB.
+ * @gdb_num_core_regs: Number of core registers accessible to GDB or 0 to infer
+ * from @gdb_core_xml_file.
  * @gdb_core_xml_file: File name for core registers GDB XML description.
  * @gdb_stop_before_watchpoint: Indicates whether GDB expects the CPU to stop
  *   before the insn which triggers a watchpoint rather than after it.
diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
index 7bea7075e1..83eafbe4b1 100644
--- a/target/s390x/cpu.h
+++ b/target/s390x/cpu.h
@@ -452,8 +452,6 @@ static inline void cpu_get_tb_cpu_state(CPUS390XState *env, 
vaddr *pc,
 #define S390_R13_REGNUM 15
 #define S390_R14_REGNUM 16
 #define S390_R15_REGNUM 17
-/* Total Core Registers. */
-#define S390_NUM_CORE_REGS 18
 
 static inline void setcc(S390CPU *cpu, uint64_t cc)
 {
diff --git a/gdbstub/gdbstub.c b/gdbstub/gdbstub.c
index 49fb23a68a..c19f1785e4 100644
--- a/gdbstub/gdbstub.c
+++ b/gdbstub/gdbstub.c
@@ -541,9 +541,12 @@ void gdb_init_cpu(CPUState *cpu)
 gdb_register_feature(cpu, 0,
  cc->gdb_read_register, cc->gdb_write_register,
  feature);
+cpu->gdb_num_regs = cpu->gdb_num_g_regs = feature->num_regs;
 }
 
-cpu->gdb_num_regs = cpu->gdb_num_g_regs = cc->gdb_num_core_regs;
+if (cc->gdb_num_core_regs) {
+cpu->gdb_num_regs = cpu->gdb_num_g_regs = cc->gdb_num_core_regs;
+}
 }
 
 void gdb_register_coprocessor(CPUState *cpu,
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 6c6c551573..6d9bf6a14e 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -2391,7 +2391,6 @@ static void arm_cpu_class_init(ObjectClass *oc, void 
*data)
 #ifndef CONFIG_USER_ONLY
 cc->sysemu_ops = &arm_sysemu_ops;
 #endif
-cc->gdb_num_core_regs = 26;
 cc->gdb_arch_name = arm_gdb_arch_name;
 cc->gdb_get_dynamic_xml = arm_gdb_get_dynamic_xml;
 cc->gdb_stop_before_watchpoint = true;
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 1cb9d5b81a..5c7a4a0bf7 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -792,7 +792,6 @@ static void aarch64_cpu_class_init(ObjectClass *oc, void 
*data)
 
 cc->gdb_read_register = aarch64_cpu_gdb_read_register;
 cc->gdb_write_register = aarch64_cpu_gdb_write_register;
-cc->gdb_num_core_regs = 34;
 cc->gdb_core_xml_file = "aarch64-core.xml";
 cc->gdb_arch_name = aarch64_gdb_arch_name;
 
diff --git a/target/avr/cpu.c b/target/avr/cpu.c
index 14d8b9d1f0..01adfb5089 100644
--- a/target/avr/cpu.c
+++ b/target/avr/cpu.c
@@ -244,7 +244,6 @@ static void avr_cpu_class_init(ObjectClass *oc, void *data)
 cc->gdb_read_register = avr_cpu_gdb_read_register;
 cc->gdb_write_register = avr_cpu_gdb_write_register;
 cc->gdb_adjust_breakpoint = avr_cpu_gdb_adjust_breakpoint;
-cc->gdb_num_core_regs = 35;
 cc->gdb_core_xml_file = "avr-cpu.xml";
 cc->tcg_ops = &avr_tcg_ops;
 }
diff --git a/target/hexagon/cpu.c b/target/hexagon/cpu.c
index 60d52e1e9d..7c1426f70c 100644
--- a/target/hexagon/cpu.c
+++ b/target/hexagon/cpu.c
@@ -385,7 +385,6 @@ static void hexagon_cpu_class_init(ObjectClass *c, void 
*data)
 cc->get_pc = hexagon_cpu_get_pc;
 cc->gdb_read_register = hexagon_gdb_read_register;
 cc->gdb_write_register = hexagon_gdb_write_register;
-cc->gdb_num_core_regs = TOTAL_PER_THREAD_REGS;
 cc->gdb_stop_before_watchpoint = true;
 cc->gdb_core_xml_file = "hexagon-core.xml";
 cc->disas_set_info = hexagon_cpu_disas_set_info;
diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index c09bab4281..6452b5439d 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -7967,10 +7967,8 @@ static void x86_cpu_common_class_init(ObjectClass *oc, 
void *data)
 cc->gdb_arch_name = x86_gdb_arch_name;
 #ifdef TARGET_X86_64
 cc->gdb_core_xml_file = "i386-64bit.xml";
-cc->gdb_num_core_regs = 66;
 #else
 cc->gdb_core_xml_file = "i386-32bit.xml";
-cc->gdb_num_cor

[PATCH v9 03/23] target/riscv: Validate misa_mxl_max only once

[Akihiko Odaki]

misa_mxl_max is now a class member and initialized only once for each
class. This also moves the initialization of gdb_core_xml_file which
will be referenced before realization in the future.

Signed-off-by: Akihiko Odaki 
---
 target/riscv/cpu.c | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c
index 1306054d27..0c087df9f9 100644
--- a/target/riscv/cpu.c
+++ b/target/riscv/cpu.c
@@ -1087,9 +1087,8 @@ static void riscv_cpu_disable_priv_spec_isa_exts(RISCVCPU 
*cpu)
 }
 }
 
-static void riscv_cpu_validate_misa_mxl(RISCVCPU *cpu)
+static void riscv_cpu_validate_misa_mxl(RISCVCPUClass *mcc)
 {
-RISCVCPUClass *mcc = RISCV_CPU_GET_CLASS(cpu);
 CPUClass *cc = CPU_CLASS(mcc);
 
 /* Validate that MISA_MXL is set properly. */
@@ -1487,8 +1486,6 @@ static void riscv_cpu_realize_tcg(DeviceState *dev, Error 
**errp)
 return;
 }
 
-riscv_cpu_validate_misa_mxl(cpu);
-
 riscv_cpu_validate_priv_spec(cpu, &local_err);
 if (local_err != NULL) {
 error_propagate(errp, local_err);
@@ -2277,6 +2274,8 @@ static void riscv_cpu_class_init(ObjectClass *c, void 
*opaque)
 
 mcc->misa_mxl_max = data->misa_mxl_max;
 mcc->misa_ext_mask = data->misa_ext_mask;
+
+riscv_cpu_validate_misa_mxl(mcc);
 }
 
 static void riscv_isa_string_ext(RISCVCPU *cpu, char **isa_str,
-- 
2.42.0

Re: [PATCH V4 01/10] accel/kvm: Extract common KVM vCPU {creation,parking} code

[Gavin Shan]

Hi Salil,

On 10/10/23 06:35, Salil Mehta wrote:

KVM vCPU creation is done once during the initialization of the VM when Qemu
thread is spawned. This is common to all the architectures.

Hot-unplug of vCPU results in destruction of the vCPU object in QOM but the
corresponding KVM vCPU object in the Host KVM is not destroyed and its
representative KVM vCPU object/context in Qemu is parked.

Refactor common logic so that some APIs could be reused by vCPU Hotplug code.

Signed-off-by: Salil Mehta 
---
  accel/kvm/kvm-all.c| 64 --
  accel/kvm/trace-events |  4 +++
  include/sysemu/kvm.h   | 16 +++
  3 files changed, 69 insertions(+), 15 deletions(-)



With the following one comment addressed:

Reviewed-by: Gavin Shan 


diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
index ff1578bb32..0dcaa15276 100644
--- a/accel/kvm/kvm-all.c
+++ b/accel/kvm/kvm-all.c
@@ -137,6 +137,7 @@ static QemuMutex kml_slots_lock;
  #define kvm_slots_unlock()  qemu_mutex_unlock(&kml_slots_lock)
  
  static void kvm_slot_init_dirty_bitmap(KVMSlot *mem);

+static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id);
  
  static inline void kvm_resample_fd_remove(int gsi)

  {
@@ -320,14 +321,53 @@ err:
  return ret;
  }
  
+void kvm_park_vcpu(CPUState *cpu)

+{
+struct KVMParkedVcpu *vcpu;
+
+trace_kvm_park_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
+
+vcpu = g_malloc0(sizeof(*vcpu));
+vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
+vcpu->kvm_fd = cpu->kvm_fd;
+QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
+}
+
+int kvm_create_vcpu(CPUState *cpu)
+{
+unsigned long vcpu_id = kvm_arch_vcpu_id(cpu);
+KVMState *s = kvm_state;
+int kvm_fd;
+
+trace_kvm_create_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
+
+/* check if the KVM vCPU already exist but is parked */
+kvm_fd = kvm_get_vcpu(s, vcpu_id);
+if (kvm_fd < 0) {
+/* vCPU not parked: create a new KVM vCPU */
+kvm_fd = kvm_vm_ioctl(s, KVM_CREATE_VCPU, vcpu_id);
+if (kvm_fd < 0) {
+error_report("KVM_CREATE_VCPU IOCTL failed for vCPU %lu", vcpu_id);
+return kvm_fd;
+}
+}
+
+cpu->kvm_fd = kvm_fd;
+cpu->kvm_state = s;
+cpu->vcpu_dirty = true;
+cpu->dirty_pages = 0;
+cpu->throttle_us_per_full = 0;
+
+return 0;
+}
+
  static int do_kvm_destroy_vcpu(CPUState *cpu)
  {
  KVMState *s = kvm_state;
  long mmap_size;
-struct KVMParkedVcpu *vcpu = NULL;
  int ret = 0;
  
-DPRINTF("kvm_destroy_vcpu\n");

+trace_kvm_destroy_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
  
  ret = kvm_arch_destroy_vcpu(cpu);

  if (ret < 0) {
@@ -353,10 +393,7 @@ static int do_kvm_destroy_vcpu(CPUState *cpu)
  }
  }
  
-vcpu = g_malloc0(sizeof(*vcpu));

-vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
-vcpu->kvm_fd = cpu->kvm_fd;
-QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
+kvm_park_vcpu(cpu);
  err:
  return ret;
  }
@@ -377,6 +414,8 @@ static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
  if (cpu->vcpu_id == vcpu_id) {
  int kvm_fd;
  
+trace_kvm_get_vcpu(vcpu_id);

+
  QLIST_REMOVE(cpu, node);
  kvm_fd = cpu->kvm_fd;
  g_free(cpu);
@@ -384,7 +423,7 @@ static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
  }
  }
  
-return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);

+return -ENOENT;
  }
  
  int kvm_init_vcpu(CPUState *cpu, Error **errp)

@@ -395,19 +434,14 @@ int kvm_init_vcpu(CPUState *cpu, Error **errp)
  
  trace_kvm_init_vcpu(cpu->cpu_index, kvm_arch_vcpu_id(cpu));
  
-ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));

+ret = kvm_create_vcpu(cpu);
  if (ret < 0) {
-error_setg_errno(errp, -ret, "kvm_init_vcpu: kvm_get_vcpu failed 
(%lu)",
+error_setg_errno(errp, -ret,
+ "kvm_init_vcpu: kvm_create_vcpu failed (%lu)",
   kvm_arch_vcpu_id(cpu));
  goto err;
  }
  
-cpu->kvm_fd = ret;

-cpu->kvm_state = s;
-cpu->vcpu_dirty = true;
-cpu->dirty_pages = 0;
-cpu->throttle_us_per_full = 0;
-
  mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
  if (mmap_size < 0) {
  ret = mmap_size;
diff --git a/accel/kvm/trace-events b/accel/kvm/trace-events
index 399aaeb0ec..08e2dc253f 100644
--- a/accel/kvm/trace-events
+++ b/accel/kvm/trace-events
@@ -9,6 +9,10 @@ kvm_device_ioctl(int fd, int type, void *arg) "dev fd %d, type 
0x%x, arg %p"
  kvm_failed_reg_get(uint64_t id, const char *msg) "Warning: Unable to retrieve ONEREG %" 
PRIu64 " from KVM: %s"
  kvm_failed_reg_set(uint64_t id, const char *msg) "Warning: Unable to set ONEREG %" 
PRIu64 " to KVM: %s"
  kvm_init_vcpu(int cpu_index, unsigned long arch_cpu_id) "index: %d id: %lu"
+kvm_create_vcpu(int cpu_index, unsigned long arch_cpu_id) "creating KVM cpu: 
cpu_index: %

[PATCH v3 0/2] target/s390x/kvm: Simplify the synchronization code

[Thomas Huth]

KVM_SYNC_GPRS, KVM_SYNC_ACRS, KVM_SYNC_CRS and KVM_SYNC_PREFIX are
available since kernel 3.10. Since we already require at least kernel
3.15 in the s390x KVM code, we can also assume that the KVM_CAP_SYNC_REGS
sync code is always possible for these registers, and remove the
related checks and fallbacks via KVM_SET_REGS and KVM_GET_REGS.

v3:
- Don't sync ACRs, CRs and prefix in kvm_arch_put_registers()
  when using level == KVM_PUT_RUNTIME_STATE
- Use g_assert(can_sync_regs(cs, KVM_SYNC_REQUIRED_REGS)) instead
  of open-coding it.

v2:
- Split the patch from v1 into two patches
- Use a #define KVM_SYNC_REQUIRED_BITS for the required sync bits
- Use memcpy() instead of for-loops for copying the registers

Thomas Huth (2):
  target/s390x/kvm: Turn KVM_CAP_SYNC_REGS into a hard requirement
  target/s390x/kvm: Simplify the GPRs, ACRs, CRs and prefix
synchronization code

 target/s390x/kvm/kvm.c | 124 -
 1 file changed, 37 insertions(+), 87 deletions(-)

-- 
2.41.0

[PATCH v3 2/2] target/s390x/kvm: Simplify the GPRs, ACRs, CRs and prefix synchronization code

[Thomas Huth]

KVM_SYNC_GPRS, KVM_SYNC_ACRS, KVM_SYNC_CRS and KVM_SYNC_PREFIX are
available since kernel 3.10. Since we already require at least kernel
3.15 in the s390x KVM code, we can also assume that the KVM_CAP_SYNC_REGS
sync code is always possible for these registers, and remove the
related checks and fallbacks via KVM_SET_REGS and KVM_GET_REGS.

Signed-off-by: Thomas Huth 
---
 target/s390x/kvm/kvm.c | 106 ++---
 1 file changed, 24 insertions(+), 82 deletions(-)

diff --git a/target/s390x/kvm/kvm.c b/target/s390x/kvm/kvm.c
index b3e2eaa2eb..dc76f63d94 100644
--- a/target/s390x/kvm/kvm.c
+++ b/target/s390x/kvm/kvm.c
@@ -138,7 +138,6 @@ const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
 KVM_CAP_LAST_INFO
 };
 
-static int cap_sync_regs;
 static int cap_async_pf;
 static int cap_mem_op;
 static int cap_mem_op_extension;
@@ -359,7 +358,6 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
 return -1;
 }
 
-cap_sync_regs = true;
 cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
 cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
 cap_mem_op_extension = kvm_check_extension(s, 
KVM_CAP_S390_MEM_OP_EXTENSION);
@@ -466,37 +464,28 @@ void kvm_s390_reset_vcpu_normal(S390CPU *cpu)
 
 static int can_sync_regs(CPUState *cs, int regs)
 {
-return cap_sync_regs && (cs->kvm_run->kvm_valid_regs & regs) == regs;
+return (cs->kvm_run->kvm_valid_regs & regs) == regs;
 }
 
+#define KVM_SYNC_REQUIRED_REGS (KVM_SYNC_GPRS | KVM_SYNC_ACRS | \
+KVM_SYNC_CRS | KVM_SYNC_PREFIX)
+
 int kvm_arch_put_registers(CPUState *cs, int level)
 {
 S390CPU *cpu = S390_CPU(cs);
 CPUS390XState *env = &cpu->env;
-struct kvm_sregs sregs;
-struct kvm_regs regs;
 struct kvm_fpu fpu = {};
 int r;
 int i;
 
+g_assert(can_sync_regs(cs, KVM_SYNC_REQUIRED_REGS));
+
 /* always save the PSW  and the GPRS*/
 cs->kvm_run->psw_addr = env->psw.addr;
 cs->kvm_run->psw_mask = env->psw.mask;
 
-if (can_sync_regs(cs, KVM_SYNC_GPRS)) {
-for (i = 0; i < 16; i++) {
-cs->kvm_run->s.regs.gprs[i] = env->regs[i];
-cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_GPRS;
-}
-} else {
-for (i = 0; i < 16; i++) {
-regs.gprs[i] = env->regs[i];
-}
-r = kvm_vcpu_ioctl(cs, KVM_SET_REGS, ®s);
-if (r < 0) {
-return r;
-}
-}
+memcpy(cs->kvm_run->s.regs.gprs, env->regs, 
sizeof(cs->kvm_run->s.regs.gprs));
+cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_GPRS;
 
 if (can_sync_regs(cs, KVM_SYNC_VRS)) {
 for (i = 0; i < 32; i++) {
@@ -529,6 +518,15 @@ int kvm_arch_put_registers(CPUState *cs, int level)
 return 0;
 }
 
+/*
+ * Access registers, control registers and the prefix - these are
+ * always available via kvm_sync_regs in the kernels that we support
+ */
+memcpy(cs->kvm_run->s.regs.acrs, env->aregs, 
sizeof(cs->kvm_run->s.regs.acrs));
+memcpy(cs->kvm_run->s.regs.crs, env->cregs, 
sizeof(cs->kvm_run->s.regs.crs));
+cs->kvm_run->s.regs.prefix = env->psa;
+cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_ACRS | KVM_SYNC_CRS | 
KVM_SYNC_PREFIX;
+
 if (can_sync_regs(cs, KVM_SYNC_ARCH0)) {
 cs->kvm_run->s.regs.cputm = env->cputm;
 cs->kvm_run->s.regs.ckc = env->ckc;
@@ -575,25 +573,6 @@ int kvm_arch_put_registers(CPUState *cs, int level)
 }
 }
 
-/* access registers and control registers*/
-if (can_sync_regs(cs, KVM_SYNC_ACRS | KVM_SYNC_CRS)) {
-for (i = 0; i < 16; i++) {
-cs->kvm_run->s.regs.acrs[i] = env->aregs[i];
-cs->kvm_run->s.regs.crs[i] = env->cregs[i];
-}
-cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_ACRS;
-cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_CRS;
-} else {
-for (i = 0; i < 16; i++) {
-sregs.acrs[i] = env->aregs[i];
-sregs.crs[i] = env->cregs[i];
-}
-r = kvm_vcpu_ioctl(cs, KVM_SET_SREGS, &sregs);
-if (r < 0) {
-return r;
-}
-}
-
 if (can_sync_regs(cs, KVM_SYNC_GSCB)) {
 memcpy(cs->kvm_run->s.regs.gscb, env->gscb, 32);
 cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_GSCB;
@@ -615,13 +594,6 @@ int kvm_arch_put_registers(CPUState *cs, int level)
 cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_DIAG318;
 }
 
-/* Finally the prefix */
-if (can_sync_regs(cs, KVM_SYNC_PREFIX)) {
-cs->kvm_run->s.regs.prefix = env->psa;
-cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_PREFIX;
-} else {
-/* prefix is only supported via sync regs */
-}
 return 0;
 }
 
@@ -629,8 +601,6 @@ int kvm_arch_get_registers(CPUState *cs)
 {
 S390CPU *cpu = S390_CPU(cs);
 CPUS390XState *env = &cpu->env;
-struct kvm_sregs sregs;
-struct kvm_regs regs;
 struct kvm_fpu fpu;
 int i, r;
 
@@ -638,37 +608,14 @@ int kvm_arch_get_registers(CPUState *cs)

[PATCH v3 1/2] target/s390x/kvm: Turn KVM_CAP_SYNC_REGS into a hard requirement

[Thomas Huth]

Since we already require at least kernel 3.15 in the s390x KVM code,
we can assume that the KVM_CAP_SYNC_REGS capability is always there.
Thus turn this into a hard requirement now.

Reviewed-by: Christian Borntraeger 
Signed-off-by: Thomas Huth 
---
 target/s390x/kvm/kvm.c | 20 ++--
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/target/s390x/kvm/kvm.c b/target/s390x/kvm/kvm.c
index bc5c56a305..b3e2eaa2eb 100644
--- a/target/s390x/kvm/kvm.c
+++ b/target/s390x/kvm/kvm.c
@@ -337,21 +337,29 @@ int kvm_arch_get_default_type(MachineState *ms)
 
 int kvm_arch_init(MachineState *ms, KVMState *s)
 {
+int required_caps[] = {
+KVM_CAP_DEVICE_CTRL,
+KVM_CAP_SYNC_REGS,
+};
+
+for (int i = 0; i < ARRAY_SIZE(required_caps); i++) {
+if (!kvm_check_extension(s, required_caps[i])) {
+error_report("KVM is missing capability #%d - "
+ "please use kernel 3.15 or newer", required_caps[i]);
+return -1;
+}
+}
+
 object_class_foreach(ccw_machine_class_foreach, TYPE_S390_CCW_MACHINE,
  false, NULL);
 
-if (!kvm_check_extension(kvm_state, KVM_CAP_DEVICE_CTRL)) {
-error_report("KVM is missing capability KVM_CAP_DEVICE_CTRL - "
- "please use kernel 3.15 or newer");
-return -1;
-}
 if (!kvm_check_extension(s, KVM_CAP_S390_COW)) {
 error_report("KVM is missing capability KVM_CAP_S390_COW - "
  "unsupported environment");
 return -1;
 }
 
-cap_sync_regs = kvm_check_extension(s, KVM_CAP_SYNC_REGS);
+cap_sync_regs = true;
 cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
 cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
 cap_mem_op_extension = kvm_check_extension(s, 
KVM_CAP_S390_MEM_OP_EXTENSION);
-- 
2.41.0

Re: [PULL 50/51] subprojects: add wrap file for libblkio

[Daniel P . Berrangé]

On Wed, Oct 11, 2023 at 07:35:24AM +0200, Philippe Mathieu-Daudé wrote:
> Hi Paolo,
> 
> On 7/9/23 14:59, Paolo Bonzini wrote:
> > This allows building libblkio at the same time as QEMU, if QEMU is
> > configured with --enable-blkio --enable-download.
> > 
> > Signed-off-by: Paolo Bonzini 
> > ---
> >   subprojects/libblkio.wrap | 6 ++
> >   1 file changed, 6 insertions(+)
> >   create mode 100644 subprojects/libblkio.wrap
> > 
> > diff --git a/subprojects/libblkio.wrap b/subprojects/libblkio.wrap
> > new file mode 100644
> > index 000..f77af72210c
> > --- /dev/null
> > +++ b/subprojects/libblkio.wrap
> > @@ -0,0 +1,6 @@
> > +[wrap-git]
> > +url = https://gitlab.com/libblkio/libblkio
> 
> Tyler noticed this project isn't mirrored on QEMU gitlab namespace.

Many other wraps aren't mirrored either, and I'm not convinced we
need to, unless the project is on an obscure self hosted git service
which is thought unreliable.

With regards,
Daniel
-- 
|: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o-https://fstop138.berrange.com :|
|: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|

Re: [PATCH v9 23/23] plugins: Support C++

[Daniel P . Berrangé]

On Wed, Oct 11, 2023 at 04:03:09PM +0900, Akihiko Odaki wrote:
> Make qemu-plugin.h consumable for C++ platform.
> 
> Signed-off-by: Akihiko Odaki 
> ---
>  docs/devel/tcg-plugins.rst |  4 
>  meson.build|  2 +-
>  include/qemu/qemu-plugin.h |  4 
>  tests/plugin/cc.cc | 16 
>  tests/plugin/meson.build   |  5 +
>  tests/tcg/Makefile.target  |  3 +--
>  6 files changed, 31 insertions(+), 3 deletions(-)
>  create mode 100644 tests/plugin/cc.cc
> 
> diff --git a/docs/devel/tcg-plugins.rst b/docs/devel/tcg-plugins.rst
> index c9f8b27590..984d8012e9 100644
> --- a/docs/devel/tcg-plugins.rst
> +++ b/docs/devel/tcg-plugins.rst
> @@ -283,6 +283,10 @@ run::
>160  1  0
>135  1  0
>  
> +- contrib/plugins/cc.cc
> +
> +A pure test plugin to ensure that the plugin API is compatible with C++.
> +

IMHO we don't need to be adding a test just to prove the
existance of the G_BEGIN_DECLS/G_END_DECLS macros in the
plugin header.

>  - contrib/plugins/hotblocks.c
>  
>  The hotblocks plugin allows you to examine the where hot paths of
> diff --git a/meson.build b/meson.build
> index 9f4c4f2f1e..c289bb8948 100644
> --- a/meson.build
> +++ b/meson.build
> @@ -20,7 +20,7 @@ config_host = keyval.load(meson.current_build_dir() / 
> 'config-host.mak')
>  
>  cc = meson.get_compiler('c')
>  all_languages = ['c']
> -if targetos == 'windows' and add_languages('cpp', required: false, native: 
> false)
> +if add_languages('cpp', required: false, native: false)
>all_languages += ['cpp']
>cxx = meson.get_compiler('cpp')
>  endif

Our goal has been to entirely eliminate C++ from our build system,
even the Windows piece will ideally go away eventually. So I'm
loathe to see us expand where we use C++ again, even if only for
a unit test.

> diff --git a/include/qemu/qemu-plugin.h b/include/qemu/qemu-plugin.h
> index 40aae8db68..55f514ca6c 100644
> --- a/include/qemu/qemu-plugin.h
> +++ b/include/qemu/qemu-plugin.h
> @@ -16,6 +16,8 @@
>  #include 
>  #include 
>  
> +G_BEGIN_DECLS
> +
>  /*
>   * For best performance, build the plugin with -fvisibility=hidden so that
>   * QEMU_PLUGIN_LOCAL is implicit. Then, just mark qemu_plugin_install with
> @@ -710,4 +712,6 @@ int qemu_plugin_find_register(unsigned int vcpu_index, 
> int file,
>   */
>  int qemu_plugin_read_register(GByteArray *buf, int reg);
>  
> +G_END_DECLS
> +
>  #endif /* QEMU_QEMU_PLUGIN_H */
> diff --git a/tests/plugin/cc.cc b/tests/plugin/cc.cc
> new file mode 100644
> index 00..297a7e4f3f
> --- /dev/null
> +++ b/tests/plugin/cc.cc
> @@ -0,0 +1,16 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +
> +#include 
> +
> +extern "C" {
> +
> +QEMU_PLUGIN_EXPORT int qemu_plugin_version = QEMU_PLUGIN_VERSION;
> +
> +QEMU_PLUGIN_EXPORT int qemu_plugin_install(qemu_plugin_id_t id,
> +   const qemu_info_t *info, int argc,
> +   char **argv)
> +{
> +return 0;
> +}
> +
> +};
> diff --git a/tests/plugin/meson.build b/tests/plugin/meson.build
> index 322cafcdf6..fed14aa0c2 100644
> --- a/tests/plugin/meson.build
> +++ b/tests/plugin/meson.build
> @@ -5,6 +5,11 @@ if get_option('plugins')
> include_directories: '../../include/qemu',
> dependencies: glib)
>endforeach
> +  if 'cpp' in all_languages
> +t += shared_module('cc', files('cc.cc'),
> +   include_directories: '../../include/qemu',
> +   dependencies: glib)
> +  endif
>  endif
>  if t.length() > 0
>alias_target('test-plugins', t)
> diff --git a/tests/tcg/Makefile.target b/tests/tcg/Makefile.target
> index 462289f47c..66a20d0f2c 100644
> --- a/tests/tcg/Makefile.target
> +++ b/tests/tcg/Makefile.target
> @@ -145,10 +145,9 @@ RUN_TESTS=$(patsubst %,run-%, $(TESTS))
>  
>  # If plugins exist also include those in the tests
>  ifeq ($(CONFIG_PLUGIN),y)
> -PLUGIN_SRC=$(SRC_PATH)/tests/plugin
>  PLUGIN_LIB=../../plugin
>  VPATH+=$(PLUGIN_LIB)
> -PLUGINS=$(patsubst %.c, lib%.so, $(notdir $(wildcard $(PLUGIN_SRC)/*.c)))
> +PLUGINS=$(notdir $(wildcard $(PLUGIN_LIB)/*.so))
>  
>  # We need to ensure expand the run-plugin-TEST-with-PLUGIN
>  # pre-requistes manually here as we can't use stems to handle it. We
> -- 
> 2.42.0
> 

With regards,
Daniel
-- 
|: https://berrange.com  -o-https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o-https://fstop138.berrange.com :|
|: https://entangle-photo.org-o-https://www.instagram.com/dberrange :|

RE: [PATCH v2] migration: refactor migration_completion

[Wang, Wei W]

On Friday, August 4, 2023 9:37 PM, Peter Xu wrote:
Fri, Aug 04, 2023 at 05:30:53PM +0800, Wei Wang wrote:
> > Current migration_completion function is a bit long. Refactor the long
> > implementation into different subfunctions:
> > - migration_completion_precopy: completion code related to precopy
> > - migration_completion_postcopy: completion code related to postcopy
> > - close_return_path_on_source: rp thread related cleanup on migration
> > completion. It is named to match with open_return_path_on_source.
> >
> > This improves readability and is easier for future updates (e.g. add
> > new subfunctions when completion code related to new features are
> > needed). No functional changes intended.
> >
> > Signed-off-by: Wei Wang 
> 
> Reviewed-by: Peter Xu 
> 

Hi Juan,
Do you think this refactoring would be good to merge or have any more comments?

Thanks,
Wei

[PULL 03/65] tests/qtest: migration: Add migrate_incoming_qmp helper

[Juan Quintela]

From: Fabiano Rosas 

file-based migration requires the target to initiate its migration after
the source has finished writing out the data in the file. Currently
there's no easy way to initiate 'migrate-incoming', allow this by
introducing migrate_incoming_qmp helper, similarly to migrate_qmp.

Also make sure migration events are enabled and wait for the incoming
migration to start before returning. This avoid a race when querying
the migration status too soon after issuing the command.

Reviewed-by: Peter Xu 
Reviewed-by: Juan Quintela 
Signed-off-by: Fabiano Rosas 
Signed-off-by: Juan Quintela 
Message-ID: <20230712190742.22294-3-faro...@suse.de>
---
 tests/qtest/migration-helpers.h |  4 
 tests/qtest/migration-helpers.c | 29 +
 2 files changed, 33 insertions(+)

diff --git a/tests/qtest/migration-helpers.h b/tests/qtest/migration-helpers.h
index 484d7c960f..57d295a4fe 100644
--- a/tests/qtest/migration-helpers.h
+++ b/tests/qtest/migration-helpers.h
@@ -23,6 +23,10 @@ bool migrate_watch_for_resume(QTestState *who, const char 
*name,
 G_GNUC_PRINTF(3, 4)
 void migrate_qmp(QTestState *who, const char *uri, const char *fmt, ...);
 
+G_GNUC_PRINTF(3, 4)
+void migrate_incoming_qmp(QTestState *who, const char *uri,
+  const char *fmt, ...);
+
 void migrate_set_capability(QTestState *who, const char *capability,
 bool value);
 
diff --git a/tests/qtest/migration-helpers.c b/tests/qtest/migration-helpers.c
index 2df198c99e..08f5ee1179 100644
--- a/tests/qtest/migration-helpers.c
+++ b/tests/qtest/migration-helpers.c
@@ -81,6 +81,35 @@ void migrate_set_capability(QTestState *who, const char 
*capability,
  capability, value);
 }
 
+void migrate_incoming_qmp(QTestState *to, const char *uri, const char *fmt, 
...)
+{
+va_list ap;
+QDict *args, *rsp, *data;
+
+va_start(ap, fmt);
+args = qdict_from_vjsonf_nofail(fmt, ap);
+va_end(ap);
+
+g_assert(!qdict_haskey(args, "uri"));
+qdict_put_str(args, "uri", uri);
+
+migrate_set_capability(to, "events", true);
+
+rsp = qtest_qmp(to, "{ 'execute': 'migrate-incoming', 'arguments': %p}",
+args);
+g_assert(qdict_haskey(rsp, "return"));
+qobject_unref(rsp);
+
+rsp = qtest_qmp_eventwait_ref(to, "MIGRATION");
+g_assert(qdict_haskey(rsp, "data"));
+
+data = qdict_get_qdict(rsp, "data");
+g_assert(qdict_haskey(data, "status"));
+g_assert_cmpstr(qdict_get_str(data, "status"), ==, "setup");
+
+qobject_unref(rsp);
+}
+
 /*
  * Note: caller is responsible to free the returned object via
  * qobject_unref() after use
-- 
2.41.0

[PULL 00/65] Migration 20231011 patches

[Juan Quintela]

The following changes since commit 0ad0d9dcd16f2ea816a413008ac5191ebef45c95:

  Merge tag 'firmware/seabios-20231010-pull-request' of 
https://gitlab.com/kraxel/qemu into staging (2023-10-10 10:22:16 -0400)

are available in the Git repository at:

  https://gitlab.com/juan.quintela/qemu.git tags/migration-20231011-pull-request

for you to fetch changes up to 5e79a4bf032213fd59aa614781751fe76584f8e8:

  migration: Add migration_rp_wait|kick() (2023-10-11 11:17:05 +0200)


Migration Pull request (20231011 edition)

Hi

In this pull request:

- Markus RDMA cleanup series
- recover fixes from peter
- migration capability from fabiano
- negative migration test from fabiano.

Please, pull.

Thanks, Juan.



Fabiano Rosas (5):
  tests/qtest: migration: Expose migrate_set_capability
  tests/qtest: migration: Add migrate_incoming_qmp helper
  tests/qtest: migration: Use migrate_incoming_qmp where appropriate
  migration: Set migration status early in incoming side
  tests/qtest: migration: Add support for negative testing of
qmp_migrate

Markus Armbruster (53):
  migration/rdma: Clean up qemu_rdma_poll()'s return type
  migration/rdma: Clean up qemu_rdma_data_init()'s return type
  migration/rdma: Clean up rdma_delete_block()'s return type
  migration/rdma: Drop fragile wr_id formatting
  migration/rdma: Consistently use uint64_t for work request IDs
  migration/rdma: Fix unwanted integer truncation
  migration/rdma: Clean up two more harmless signed vs. unsigned issues
  migration/rdma: Give qio_channel_rdma_source_funcs internal linkage
  migration/rdma: Fix qemu_rdma_accept() to return failure on errors
  migration/rdma: Put @errp parameter last
  migration/rdma: Eliminate error_propagate()
  migration/rdma: Drop rdma_add_block() error handling
  migration/rdma: Drop qemu_rdma_search_ram_block() error handling
  migration/rdma: Make qemu_rdma_buffer_mergeable() return bool
  migration/rdma: Use bool for two RDMAContext flags
  migration/rdma: Fix or document problematic uses of errno
  migration/rdma: Ditch useless numeric error codes in error messages
  migration/rdma: Fix io_writev(), io_readv() methods to obey contract
  migration/rdma: Replace dangerous macro CHECK_ERROR_STATE()
  migration/rdma: Fix qemu_rdma_broken_ipv6_kernel() to set error
  migration/rdma: Fix qemu_get_cm_event_timeout() to always set error
  migration/rdma: Drop dead qemu_rdma_data_init() code for !@host_port
  migration/rdma: Fix QEMUFileHooks method return values
  migration/rdma: Fix rdma_getaddrinfo() error checking
  migration/rdma: Return -1 instead of negative errno code
  migration/rdma: Dumb down remaining int error values to -1
  migration/rdma: Replace int error_state by bool errored
  migration/rdma: Drop superfluous assignments to @ret
  migration/rdma: Check negative error values the same way everywhere
  migration/rdma: Plug a memory leak and improve a message
  migration/rdma: Delete inappropriate error_report() in macro ERROR()
  migration/rdma: Retire macro ERROR()
  migration/rdma: Fix error handling around rdma_getaddrinfo()
  migration/rdma: Drop "@errp is clear" guards around error_setg()
  migration/rdma: Convert qemu_rdma_exchange_recv() to Error
  migration/rdma: Convert qemu_rdma_exchange_send() to Error
  migration/rdma: Convert qemu_rdma_exchange_get_response() to Error
  migration/rdma: Convert qemu_rdma_reg_whole_ram_blocks() to Error
  migration/rdma: Convert qemu_rdma_write_flush() to Error
  migration/rdma: Convert qemu_rdma_write_one() to Error
  migration/rdma: Convert qemu_rdma_write() to Error
  migration/rdma: Convert qemu_rdma_post_send_control() to Error
  migration/rdma: Convert qemu_rdma_post_recv_control() to Error
  migration/rdma: Convert qemu_rdma_alloc_pd_cq() to Error
  migration/rdma: Silence qemu_rdma_resolve_host()
  migration/rdma: Silence qemu_rdma_connect()
  migration/rdma: Silence qemu_rdma_reg_control()
  migration/rdma: Don't report received completion events as error
  migration/rdma: Silence qemu_rdma_block_for_wrid()
  migration/rdma: Silence qemu_rdma_register_and_get_keys()
  migration/rdma: Downgrade qemu_rdma_cleanup() errors to warnings
  migration/rdma: Use error_report() & friends instead of stderr
  migration/rdma: Replace flawed device detail dump by tracing

Peter Xu (7):
  migration/qmp: Fix crash on setting tls-authz with null
  migration: Allow RECOVER->PAUSED convertion for dest qemu
  migration: Display error in query-migrate irrelevant of status
  migration: Introduce migrate_has_error()
  qemufile: Always return a verbose error
  migration: Remember num of ramblocks to sync during recovery
  migration: Add migration_rp_wait|kick()

 qapi/migration.json   |5 +-
 migration/migration.h |   16 +
 tests/qtest/libqtest.h|   28 +
 tests/qtest/migrat

[PULL 05/65] migration: Set migration status early in incoming side

[Juan Quintela]

From: Fabiano Rosas 

We are sending a migration event of MIGRATION_STATUS_SETUP at
qemu_start_incoming_migration but never actually setting the state.

This creates a window between qmp_migrate_incoming and
process_incoming_migration_co where the migration status is still
MIGRATION_STATUS_NONE. Calling query-migrate during this time will
return an empty response even though the incoming migration command
has already been issued.

Commit 7cf1fe6d68 ("migration: Add migration events on target side")
has added support to the 'events' capability to the incoming part of
migration, but chose to send the SETUP event without setting the
state. I'm assuming this was a mistake.

This introduces a change in behavior, any QMP client waiting for the
SETUP event will hang, unless it has previously enabled the 'events'
capability. Having the capability enabled is sufficient to continue to
receive the event.

Reviewed-by: Peter Xu 
Reviewed-by: Juan Quintela 
Signed-off-by: Fabiano Rosas 
Signed-off-by: Juan Quintela 
Message-ID: <20230712190742.22294-5-faro...@suse.de>
---
 migration/migration.c | 7 +--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/migration/migration.c b/migration/migration.c
index 585d3c8f55..2057e42134 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -431,13 +431,16 @@ void migrate_add_address(SocketAddress *address)
 static void qemu_start_incoming_migration(const char *uri, Error **errp)
 {
 const char *p = NULL;
+MigrationIncomingState *mis = migration_incoming_get_current();
 
 /* URI is not suitable for migration? */
 if (!migration_channels_and_uri_compatible(uri, errp)) {
 return;
 }
 
-qapi_event_send_migration(MIGRATION_STATUS_SETUP);
+migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
+  MIGRATION_STATUS_SETUP);
+
 if (strstart(uri, "tcp:", &p) ||
 strstart(uri, "unix:", NULL) ||
 strstart(uri, "vsock:", NULL)) {
@@ -531,7 +534,7 @@ process_incoming_migration_co(void *opaque)
 
 mis->largest_page_size = qemu_ram_pagesize_largest();
 postcopy_state_set(POSTCOPY_INCOMING_NONE);
-migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
+migrate_set_state(&mis->state, MIGRATION_STATUS_SETUP,
   MIGRATION_STATUS_ACTIVE);
 
 mis->loadvm_co = qemu_coroutine_self();
-- 
2.41.0

[PULL 09/65] migration/rdma: Clean up qemu_rdma_data_init()'s return type

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_data_init() return type is void *.  It actually returns
RDMAContext *, and all its callers assign the value to an
RDMAContext *.  Unclean.

Return RDMAContext * instead.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-3-arm...@redhat.com>
---
 migration/rdma.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index e72864d1cf..1432fb80ec 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2759,7 +2759,7 @@ static void qemu_rdma_return_path_dest_init(RDMAContext 
*rdma_return_path,
 rdma_return_path->is_return_path = true;
 }
 
-static void *qemu_rdma_data_init(const char *host_port, Error **errp)
+static RDMAContext *qemu_rdma_data_init(const char *host_port, Error **errp)
 {
 RDMAContext *rdma = NULL;
 InetSocketAddress *addr;
-- 
2.41.0

[PULL 25/65] migration/rdma: Fix io_writev(), io_readv() methods to obey contract

[Juan Quintela]

From: Markus Armbruster 

QIOChannelClass methods qio_channel_rdma_readv() and
qio_channel_rdma_writev() violate their method contract when
rdma->error_state is non-zero:

1. They return whatever is in rdma->error_state then.  Only -1 will be
   fine.  -2 will be misinterpreted as "would block".  Anything less
   than -2 isn't defined in the contract.  A positive value would be
   misinterpreted as success, but I believe that's not actually
   possible.

2. They neglect to set an error then.  If something up the call stack
   dereferences the error when failure is returned, it will crash.  If
   it ignores the return value and checks the error instead, it will
   miss the error.

Crap like this happens when return statements hide in macros,
especially when their uses are far away from the definition.

I elected not to investigate how callers are impacted.

Expand the two bad macro uses, so we can set an error and return -1.
The next commit will then get rid of the macro altogether.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-19-arm...@redhat.com>
---
 migration/rdma.c | 12 ++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index c4197c6437..18be228e3b 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2810,7 +2810,11 @@ static ssize_t qio_channel_rdma_writev(QIOChannel *ioc,
 return -1;
 }
 
-CHECK_ERROR_STATE();
+if (rdma->error_state) {
+error_setg(errp,
+   "RDMA is in an error state waiting migration to abort!");
+return -1;
+}
 
 /*
  * Push out any writes that
@@ -2896,7 +2900,11 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
 return -1;
 }
 
-CHECK_ERROR_STATE();
+if (rdma->error_state) {
+error_setg(errp,
+   "RDMA is in an error state waiting migration to abort!");
+return -1;
+}
 
 for (i = 0; i < niov; i++) {
 size_t want = iov[i].iov_len;
-- 
2.41.0

[PULL 15/65] migration/rdma: Give qio_channel_rdma_source_funcs internal linkage

[Juan Quintela]

From: Markus Armbruster 

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-9-arm...@redhat.com>
---
 migration/rdma.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 1aac50b0de..1159f990af 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -3075,7 +3075,7 @@ qio_channel_rdma_source_finalize(GSource *source)
 object_unref(OBJECT(ssource->rioc));
 }
 
-GSourceFuncs qio_channel_rdma_source_funcs = {
+static GSourceFuncs qio_channel_rdma_source_funcs = {
 qio_channel_rdma_source_prepare,
 qio_channel_rdma_source_check,
 qio_channel_rdma_source_dispatch,
-- 
2.41.0

[PULL 20/65] migration/rdma: Drop qemu_rdma_search_ram_block() error handling

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_search_ram_block() can't fail.  Return void, and drop the
unreachable error handling.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-14-arm...@redhat.com>
---
 migration/rdma.c | 24 
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 466725dbf0..b412dad542 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1231,15 +1231,13 @@ static int qemu_rdma_reg_whole_ram_blocks(RDMAContext 
*rdma)
  *
  * Once the block is found, also identify which 'chunk' within that
  * block that the page belongs to.
- *
- * This search cannot fail or the migration will fail.
  */
-static int qemu_rdma_search_ram_block(RDMAContext *rdma,
-  uintptr_t block_offset,
-  uint64_t offset,
-  uint64_t length,
-  uint64_t *block_index,
-  uint64_t *chunk_index)
+static void qemu_rdma_search_ram_block(RDMAContext *rdma,
+   uintptr_t block_offset,
+   uint64_t offset,
+   uint64_t length,
+   uint64_t *block_index,
+   uint64_t *chunk_index)
 {
 uint64_t current_addr = block_offset + offset;
 RDMALocalBlock *block = g_hash_table_lookup(rdma->blockmap,
@@ -1251,8 +1249,6 @@ static int qemu_rdma_search_ram_block(RDMAContext *rdma,
 *block_index = block->index;
 *chunk_index = ram_chunk_index(block->local_host_addr,
 block->local_host_addr + (current_addr - block->offset));
-
-return 0;
 }
 
 /*
@@ -2341,12 +2337,8 @@ static int qemu_rdma_write(RDMAContext *rdma,
 rdma->current_length = 0;
 rdma->current_addr = current_addr;
 
-ret = qemu_rdma_search_ram_block(rdma, block_offset,
- offset, len, &index, &chunk);
-if (ret) {
-error_report("ram block search failed");
-return ret;
-}
+qemu_rdma_search_ram_block(rdma, block_offset,
+   offset, len, &index, &chunk);
 rdma->current_index = index;
 rdma->current_chunk = chunk;
 }
-- 
2.41.0

[PULL 17/65] migration/rdma: Put @errp parameter last

[Juan Quintela]

From: Markus Armbruster 

include/qapi/error.h demands:

 * - Functions that use Error to report errors have an Error **errp
 *   parameter.  It should be the last parameter, except for functions
 *   taking variable arguments.

qemu_rdma_connect() does not conform.  Clean it up.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-11-arm...@redhat.com>
---
 migration/rdma.c | 7 ---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 8fd1b314b5..bc6d8248f2 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2552,7 +2552,8 @@ static int qemu_get_cm_event_timeout(RDMAContext *rdma,
 }
 }
 
-static int qemu_rdma_connect(RDMAContext *rdma, Error **errp, bool return_path)
+static int qemu_rdma_connect(RDMAContext *rdma, bool return_path,
+ Error **errp)
 {
 RDMACapabilities cap = {
 .version = RDMA_CONTROL_VERSION_CURRENT,
@@ -4183,7 +4184,7 @@ void rdma_start_outgoing_migration(void *opaque,
 }
 
 trace_rdma_start_outgoing_migration_after_rdma_source_init();
-ret = qemu_rdma_connect(rdma, errp, false);
+ret = qemu_rdma_connect(rdma, false, errp);
 
 if (ret) {
 goto err;
@@ -4204,7 +4205,7 @@ void rdma_start_outgoing_migration(void *opaque,
 goto return_path_err;
 }
 
-ret = qemu_rdma_connect(rdma_return_path, errp, true);
+ret = qemu_rdma_connect(rdma_return_path, true, errp);
 
 if (ret) {
 goto return_path_err;
-- 
2.41.0

[PULL 26/65] migration/rdma: Replace dangerous macro CHECK_ERROR_STATE()

[Juan Quintela]

From: Markus Armbruster 

Hiding return statements in macros is a bad idea.  Use a function
instead, and open code the return part.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-20-arm...@redhat.com>
---
 migration/rdma.c | 43 +++
 1 file changed, 27 insertions(+), 16 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 18be228e3b..30e2c817f2 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -85,18 +85,6 @@
  */
 static uint32_t known_capabilities = RDMA_CAPABILITY_PIN_ALL;
 
-#define CHECK_ERROR_STATE() \
-do { \
-if (rdma->error_state) { \
-if (!rdma->error_reported) { \
-error_report("RDMA is in an error state waiting migration" \
-" to abort!"); \
-rdma->error_reported = true; \
-} \
-return rdma->error_state; \
-} \
-} while (0)
-
 /*
  * A work request ID is 64-bits and we split up these bits
  * into 3 parts:
@@ -451,6 +439,16 @@ typedef struct QEMU_PACKED {
 uint64_t chunks;/* how many sequential chunks to register */
 } RDMARegister;
 
+static int check_error_state(RDMAContext *rdma)
+{
+if (rdma->error_state && !rdma->error_reported) {
+error_report("RDMA is in an error state waiting migration"
+ " to abort!");
+rdma->error_reported = true;
+}
+return rdma->error_state;
+}
+
 static void register_to_network(RDMAContext *rdma, RDMARegister *reg)
 {
 RDMALocalBlock *local_block;
@@ -3268,7 +3266,10 @@ static int qemu_rdma_save_page(QEMUFile *f, ram_addr_t 
block_offset,
 return -EIO;
 }
 
-CHECK_ERROR_STATE();
+ret = check_error_state(rdma);
+if (ret) {
+return ret;
+}
 
 qemu_fflush(f);
 
@@ -3574,7 +3575,10 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 return -EIO;
 }
 
-CHECK_ERROR_STATE();
+ret = check_error_state(rdma);
+if (ret) {
+return ret;
+}
 
 local = &rdma->local_ram_blocks;
 do {
@@ -3878,6 +3882,7 @@ static int qemu_rdma_registration_start(QEMUFile *f,
 {
 QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
 RDMAContext *rdma;
+int ret;
 
 if (migration_in_postcopy()) {
 return 0;
@@ -3889,7 +3894,10 @@ static int qemu_rdma_registration_start(QEMUFile *f,
 return -EIO;
 }
 
-CHECK_ERROR_STATE();
+ret = check_error_state(rdma);
+if (ret) {
+return ret;
+}
 
 trace_qemu_rdma_registration_start(flags);
 qemu_put_be64(f, RAM_SAVE_FLAG_HOOK);
@@ -3920,7 +3928,10 @@ static int qemu_rdma_registration_stop(QEMUFile *f,
 return -EIO;
 }
 
-CHECK_ERROR_STATE();
+ret = check_error_state(rdma);
+if (ret) {
+return ret;
+}
 
 qemu_fflush(f);
 ret = qemu_rdma_drain_cq(rdma);
-- 
2.41.0

[PULL 22/65] migration/rdma: Use bool for two RDMAContext flags

[Juan Quintela]

From: Markus Armbruster 

@error_reported and @received_error are flags.  The latter is even
assigned bool true.  Change them from int to bool.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-16-arm...@redhat.com>
---
 migration/rdma.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 2e62d2cd0a..dffca30382 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -91,7 +91,7 @@ static uint32_t known_capabilities = RDMA_CAPABILITY_PIN_ALL;
 if (!rdma->error_reported) { \
 error_report("RDMA is in an error state waiting migration" \
 " to abort!"); \
-rdma->error_reported = 1; \
+rdma->error_reported = true; \
 } \
 return rdma->error_state; \
 } \
@@ -365,8 +365,8 @@ typedef struct RDMAContext {
  * and remember the error state.
  */
 int error_state;
-int error_reported;
-int received_error;
+bool error_reported;
+bool received_error;
 
 /*
  * Description of ram blocks used throughout the code.
-- 
2.41.0

[PULL 33/65] migration/rdma: Dumb down remaining int error values to -1

[Juan Quintela]

From: Markus Armbruster 

This is just to make the error value more obvious.  Callers don't
mind.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-27-arm...@redhat.com>
---
 migration/rdma.c | 45 +++--
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index bc39b7ab2e..3c7518d65a 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1434,7 +1434,7 @@ static int qemu_rdma_unregister_waiting(RDMAContext *rdma)
 ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) ®,
 &resp, NULL, NULL);
 if (ret < 0) {
-return ret;
+return -1;
 }
 
 trace_qemu_rdma_unregister_waiting_complete(chunk);
@@ -1475,7 +1475,7 @@ static int qemu_rdma_poll(RDMAContext *rdma, struct 
ibv_cq *cq,
 
 if (ret < 0) {
 error_report("ibv_poll_cq failed");
-return ret;
+return -1;
 }
 
 wr_id = wc.wr_id & RDMA_WRID_TYPE_MASK;
@@ -1604,7 +1604,7 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma,
 if (rdma->received_error) {
 return -1;
 }
-return rdma->error_state;
+return -!!rdma->error_state;
 }
 
 static struct ibv_comp_channel *to_channel(RDMAContext *rdma, uint64_t wrid)
@@ -1649,7 +1649,7 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma,
 while (wr_id != wrid_requested) {
 ret = qemu_rdma_poll(rdma, poll_cq, &wr_id_in, byte_len);
 if (ret < 0) {
-return ret;
+return -1;
 }
 
 wr_id = wr_id_in & RDMA_WRID_TYPE_MASK;
@@ -1723,7 +1723,7 @@ err_block_for_wrid:
 }
 
 rdma->error_state = ret;
-return ret;
+return -1;
 }
 
 /*
@@ -1778,9 +1778,10 @@ static int qemu_rdma_post_send_control(RDMAContext 
*rdma, uint8_t *buf,
 ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_SEND_CONTROL, NULL);
 if (ret < 0) {
 error_report("rdma migration: send polling control error");
+return -1;
 }
 
-return ret;
+return 0;
 }
 
 /*
@@ -1822,7 +1823,7 @@ static int qemu_rdma_exchange_get_response(RDMAContext 
*rdma,
 
 if (ret < 0) {
 error_report("rdma migration: recv polling control error!");
-return ret;
+return -1;
 }
 
 network_to_control((void *) rdma->wr_data[idx].control);
@@ -1902,7 +1903,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
   RDMA_CONTROL_READY,
   RDMA_WRID_READY);
 if (ret < 0) {
-return ret;
+return -1;
 }
 }
 
@@ -1914,7 +1915,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 if (ret) {
 error_report("rdma migration: error posting"
 " extra control recv for anticipated result!");
-return ret;
+return -1;
 }
 }
 
@@ -1924,7 +1925,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
 if (ret) {
 error_report("rdma migration: error posting first control recv!");
-return ret;
+return -1;
 }
 
 /*
@@ -1934,7 +1935,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 
 if (ret < 0) {
 error_report("Failed to send control buffer!");
-return ret;
+return -1;
 }
 
 /*
@@ -1945,7 +1946,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 trace_qemu_rdma_exchange_send_issue_callback();
 ret = callback(rdma);
 if (ret < 0) {
-return ret;
+return -1;
 }
 }
 
@@ -1954,7 +1955,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
   resp->type, RDMA_WRID_DATA);
 
 if (ret < 0) {
-return ret;
+return -1;
 }
 
 qemu_rdma_move_header(rdma, RDMA_WRID_DATA, resp);
@@ -1990,7 +1991,7 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
 
 if (ret < 0) {
 error_report("Failed to send control buffer!");
-return ret;
+return -1;
 }
 
 /*
@@ -2000,7 +2001,7 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
   expecting, RDMA_WRID_READY);
 
 if (ret < 0) {
-return ret;
+return -1;
 }
 
 qemu_rdma_move_header(rdma, RDMA_WRID_READY, head);
@@ -2011,7 +2012,7 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
 ret = qemu_rdma_post_recv_control(rdma, R

[PULL 64/65] migration: Remember num of ramblocks to sync during recovery

[Juan Quintela]

From: Peter Xu 

Instead of only relying on the count of rp_sem, make the counter be part of
RAMState so it can be used in both threads to synchronize on the process.

rp_sem will be further reused in follow up patches, as a way to kick the
main thread, e.g., on recovery failures.

Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Peter Xu 
Signed-off-by: Juan Quintela 
Message-ID: <20231004220240.167175-7-pet...@redhat.com>
---
 migration/ram.c | 17 ++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/migration/ram.c b/migration/ram.c
index e4bfd39f08..6c40d9af0c 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -394,6 +394,14 @@ struct RAMState {
 /* Queue of outstanding page requests from the destination */
 QemuMutex src_page_req_mutex;
 QSIMPLEQ_HEAD(, RAMSrcPageRequest) src_page_requests;
+
+/*
+ * This is only used when postcopy is in recovery phase, to communicate
+ * between the migration thread and the return path thread on dirty
+ * bitmap synchronizations.  This field is unused in other stages of
+ * RAM migration.
+ */
+unsigned int postcopy_bmap_sync_requested;
 };
 typedef struct RAMState RAMState;
 
@@ -4119,20 +4127,20 @@ static int ram_dirty_bitmap_sync_all(MigrationState *s, 
RAMState *rs)
 {
 RAMBlock *block;
 QEMUFile *file = s->to_dst_file;
-int ramblock_count = 0;
 
 trace_ram_dirty_bitmap_sync_start();
 
+qatomic_set(&rs->postcopy_bmap_sync_requested, 0);
 RAMBLOCK_FOREACH_NOT_IGNORED(block) {
 qemu_savevm_send_recv_bitmap(file, block->idstr);
 trace_ram_dirty_bitmap_request(block->idstr);
-ramblock_count++;
+qatomic_inc(&rs->postcopy_bmap_sync_requested);
 }
 
 trace_ram_dirty_bitmap_sync_wait();
 
 /* Wait until all the ramblocks' dirty bitmap synced */
-while (ramblock_count--) {
+while (qatomic_read(&rs->postcopy_bmap_sync_requested)) {
 qemu_sem_wait(&s->rp_state.rp_sem);
 }
 
@@ -4159,6 +4167,7 @@ int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock 
*block)
 unsigned long *le_bitmap, nbits = block->used_length >> TARGET_PAGE_BITS;
 uint64_t local_size = DIV_ROUND_UP(nbits, 8);
 uint64_t size, end_mark;
+RAMState *rs = ram_state;
 
 trace_ram_dirty_bitmap_reload_begin(block->idstr);
 
@@ -4225,6 +4234,8 @@ int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock 
*block)
 /* We'll recalculate migration_dirty_pages in ram_state_resume_prepare(). 
*/
 trace_ram_dirty_bitmap_reload_complete(block->idstr);
 
+qatomic_dec(&rs->postcopy_bmap_sync_requested);
+
 /*
  * We succeeded to sync bitmap for current ramblock. If this is
  * the last one to sync, we need to notify the main send thread.
-- 
2.41.0

[PULL 37/65] migration/rdma: Plug a memory leak and improve a message

[Juan Quintela]

From: Markus Armbruster 

When migration capability @rdma-pin-all is true, but the server cannot
honor it, qemu_rdma_connect() calls macro ERROR(), then returns
success.

ERROR() sets an error.  Since qemu_rdma_connect() returns success, its
caller rdma_start_outgoing_migration() duly assumes @errp is still
clear.  The Error object leaks.

ERROR() additionally reports the situation to the user as an error:

RDMA ERROR: Server cannot support pinning all memory. Will register memory 
dynamically.

Is this an error or not?  It actually isn't; we disable @rdma-pin-all
and carry on.  "Correcting" the user's configuration decisions that
way feels problematic, but that's a topic for another day.

Replace ERROR() by warn_report().  This plugs the memory leak, and
emits a clearer message to the user.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-31-arm...@redhat.com>
---
 migration/rdma.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index b0125b01cf..00e3c430f4 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2637,8 +2637,8 @@ static int qemu_rdma_connect(RDMAContext *rdma, bool 
return_path,
  * and disable them otherwise.
  */
 if (rdma->pin_all && !(cap.flags & RDMA_CAPABILITY_PIN_ALL)) {
-ERROR(errp, "Server cannot support pinning all memory. "
-"Will register memory dynamically.");
+warn_report("RDMA: Server cannot support pinning all memory. "
+"Will register memory dynamically.");
 rdma->pin_all = false;
 }
 
-- 
2.41.0

Re: [PATCH RFC v4 4/9] target/loongarch: Implement kvm get/set registers

[lixianglai]

Hi Philippe Mathieu-Daudé:


Hi Li and Zhao,

On 9/10/23 11:01, xianglai li wrote:

From: Tianrui Zhao 

Implement kvm_arch_get/set_registers interfaces, many regs
can be get/set in the function, such as core regs, csr regs,
fpu regs, mp state, etc.

Cc: "Michael S. Tsirkin" 
Cc: Cornelia Huck 
Cc: Paolo Bonzini 
Cc: "Marc-André Lureau" 
Cc: "Daniel P. Berrangé" 
Cc: Thomas Huth 
Cc: "Philippe Mathieu-Daudé" 
Cc: Richard Henderson 
Cc: Peter Maydell 
Cc: Bibo Mao 
Cc: Song Gao 
Cc: Xiaojuan Yang 
Cc: Tianrui Zhao 

Signed-off-by: Tianrui Zhao 
Signed-off-by: xianglai li 
---
  meson.build   |   1 +
  target/loongarch/cpu.c    |   3 +
  target/loongarch/cpu.h    |   2 +
  target/loongarch/kvm.c    | 406 +-
  target/loongarch/trace-events |  13 ++
  target/loongarch/trace.h  |   1 +
  6 files changed, 424 insertions(+), 2 deletions(-)
  create mode 100644 target/loongarch/trace-events
  create mode 100644 target/loongarch/trace.h




+static int kvm_larch_getq(CPUState *cs, uint64_t reg_id,
+ uint64_t *addr)
+{
+    struct kvm_one_reg csrreg = {
+    .id = reg_id,
+    .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &csrreg);
+}


This is kvm_get_one_reg().


I'll replace kvm_larch_getq() with kvm_get_one_reg().






+static int kvm_larch_putq(CPUState *cs, uint64_t reg_id,
+ uint64_t *addr)
+{
+    struct kvm_one_reg csrreg = {
+    .id = reg_id,
+    .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &csrreg);
+}


This is kvm_set_one_reg().


I'll replace kvm_larch_putq() with kvm_set_one_reg().

Thanks,

Xianglai.



+
+#define KVM_GET_ONE_UREG64(cs, ret, regidx, addr) \
+ ({    \
+    err = kvm_larch_getq(cs, KVM_IOC_CSRID(regidx), addr);    \
+    if (err < 0) {    \
+    ret = err;    \
+    trace_kvm_failed_get_csr(regidx, strerror(errno));    \
+ } \
+    })
+
+#define KVM_PUT_ONE_UREG64(cs, ret, regidx, addr) \
+ ({    \
+    err = kvm_larch_putq(cs, KVM_IOC_CSRID(regidx), addr);    \
+    if (err < 0) {    \
+    ret = err;    \
+    trace_kvm_failed_put_csr(regidx, strerror(errno));    \
+ } \
+    })

[PULL 30/65] migration/rdma: Fix QEMUFileHooks method return values

[Juan Quintela]

From: Markus Armbruster 

The QEMUFileHooks methods don't come with a written contract.  Digging
through the code calling them, we find:

* save_page():

  Negative values RAM_SAVE_CONTROL_DELAYED and
  RAM_SAVE_CONTROL_NOT_SUPP are special.  Any other negative value is
  an unspecified error.

  qemu_rdma_save_page() returns -EIO or rdma->error_state on error.  I
  believe the latter is always negative.  Nothing stops either of them
  to clash with the special values, though.  Feels unlikely, but fix
  it anyway to return only the special values and -1.

* before_ram_iterate(), after_ram_iterate():

  Negative value means error.  qemu_rdma_registration_start() and
  qemu_rdma_registration_stop() comply as far as I can tell.  Make
  them comply *obviously*, by returning -1 on error.

* hook_ram_load:

  Negative value means error.  rdma_load_hook() already returns -1 on
  error.  Leave it alone.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-24-arm...@redhat.com>
---
 migration/rdma.c | 79 +++-
 1 file changed, 37 insertions(+), 42 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 55eb8222ea..974edde6a3 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -3268,12 +3268,11 @@ static int qemu_rdma_save_page(QEMUFile *f, ram_addr_t 
block_offset,
 rdma = qatomic_rcu_read(&rioc->rdmaout);
 
 if (!rdma) {
-return -EIO;
+return -1;
 }
 
-ret = check_error_state(rdma);
-if (ret) {
-return ret;
+if (check_error_state(rdma)) {
+return -1;
 }
 
 qemu_fflush(f);
@@ -3329,9 +3328,10 @@ static int qemu_rdma_save_page(QEMUFile *f, ram_addr_t 
block_offset,
 }
 
 return RAM_SAVE_CONTROL_DELAYED;
+
 err:
 rdma->error_state = ret;
-return ret;
+return -1;
 }
 
 static void rdma_accept_incoming_migration(void *opaque);
@@ -3577,12 +3577,11 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 rdma = qatomic_rcu_read(&rioc->rdmain);
 
 if (!rdma) {
-return -EIO;
+return -1;
 }
 
-ret = check_error_state(rdma);
-if (ret) {
-return ret;
+if (check_error_state(rdma)) {
+return -1;
 }
 
 local = &rdma->local_ram_blocks;
@@ -3615,7 +3614,7 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
  (unsigned int)comp->block_idx,
  rdma->local_ram_blocks.nb_blocks);
 ret = -EIO;
-goto out;
+goto err;
 }
 block = &(rdma->local_ram_blocks.block[comp->block_idx]);
 
@@ -3627,7 +3626,7 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 
 case RDMA_CONTROL_REGISTER_FINISHED:
 trace_qemu_rdma_registration_handle_finished();
-goto out;
+return 0;
 
 case RDMA_CONTROL_RAM_BLOCKS_REQUEST:
 trace_qemu_rdma_registration_handle_ram_blocks();
@@ -3648,7 +3647,7 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 if (ret) {
 error_report("rdma migration: error dest "
 "registering ram blocks");
-goto out;
+goto err;
 }
 }
 
@@ -3687,7 +3686,7 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 
 if (ret < 0) {
 error_report("rdma migration: error sending remote info");
-goto out;
+goto err;
 }
 
 break;
@@ -3714,7 +3713,7 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
  (unsigned int)reg->current_index,
  rdma->local_ram_blocks.nb_blocks);
 ret = -ENOENT;
-goto out;
+goto err;
 }
 block = &(rdma->local_ram_blocks.block[reg->current_index]);
 if (block->is_ram_block) {
@@ -3724,7 +3723,7 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 block->block_name, block->offset,
 reg->key.current_addr);
 ret = -ERANGE;
-goto out;
+goto err;
 }
 host_addr = (block->local_host_addr +
 (reg->key.current_addr - block->offset));
@@ -3740,7 +3739,7 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 " chunk: %" PRIx64,
 block->block_name, reg->key.chunk);
 ret = -ERANGE;
-goto out;
+goto err;
 }
 }
 chunk_start = ram_chunk_start(block, chunk);
@@ -37

[PULL 34/65] migration/rdma: Replace int error_state by bool errored

[Juan Quintela]

From: Markus Armbruster 

All we do with the value of RDMAContext member @error_state is test
whether it's zero.  Change to bool and rename to @errored.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-28-arm...@redhat.com>
---
 migration/rdma.c | 66 
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 3c7518d65a..a57ec3791a 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -352,7 +352,7 @@ typedef struct RDMAContext {
  * memory registration, then do not attempt any future work
  * and remember the error state.
  */
-int error_state;
+bool errored;
 bool error_reported;
 bool received_error;
 
@@ -439,14 +439,14 @@ typedef struct QEMU_PACKED {
 uint64_t chunks;/* how many sequential chunks to register */
 } RDMARegister;
 
-static int check_error_state(RDMAContext *rdma)
+static bool rdma_errored(RDMAContext *rdma)
 {
-if (rdma->error_state && !rdma->error_reported) {
+if (rdma->errored && !rdma->error_reported) {
 error_report("RDMA is in an error state waiting migration"
  " to abort!");
 rdma->error_reported = true;
 }
-return rdma->error_state;
+return rdma->errored;
 }
 
 static void register_to_network(RDMAContext *rdma, RDMARegister *reg)
@@ -1547,7 +1547,7 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma,
  * But we need to be able to handle 'cancel' or an error
  * without hanging forever.
  */
-while (!rdma->error_state  && !rdma->received_error) {
+while (!rdma->errored && !rdma->received_error) {
 GPollFD pfds[2];
 pfds[0].fd = comp_channel->fd;
 pfds[0].events = G_IO_IN | G_IO_HUP | G_IO_ERR;
@@ -1604,7 +1604,7 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma,
 if (rdma->received_error) {
 return -1;
 }
-return -!!rdma->error_state;
+return -rdma->errored;
 }
 
 static struct ibv_comp_channel *to_channel(RDMAContext *rdma, uint64_t wrid)
@@ -1722,7 +1722,7 @@ err_block_for_wrid:
 ibv_ack_cq_events(cq, num_cq_events);
 }
 
-rdma->error_state = ret;
+rdma->errored = true;
 return -1;
 }
 
@@ -2386,7 +2386,7 @@ static void qemu_rdma_cleanup(RDMAContext *rdma)
 int idx;
 
 if (rdma->cm_id && rdma->connected) {
-if ((rdma->error_state ||
+if ((rdma->errored ||
  migrate_get_current()->state == MIGRATION_STATUS_CANCELLING) &&
 !rdma->received_error) {
 RDMAControlHeader head = { .len = 0,
@@ -2672,14 +2672,14 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error 
**errp)
 
 if (!rdma->host || !rdma->host[0]) {
 ERROR(errp, "RDMA host is not set!");
-rdma->error_state = -EINVAL;
+rdma->errored = true;
 return -1;
 }
 /* create CM channel */
 rdma->channel = rdma_create_event_channel();
 if (!rdma->channel) {
 ERROR(errp, "could not create rdma event channel");
-rdma->error_state = -EINVAL;
+rdma->errored = true;
 return -1;
 }
 
@@ -2737,7 +2737,7 @@ err_dest_init_bind_addr:
 err_dest_init_create_listen_id:
 rdma_destroy_event_channel(rdma->channel);
 rdma->channel = NULL;
-rdma->error_state = ret;
+rdma->errored = true;
 return -1;
 
 }
@@ -2812,7 +2812,7 @@ static ssize_t qio_channel_rdma_writev(QIOChannel *ioc,
 return -1;
 }
 
-if (rdma->error_state) {
+if (rdma->errored) {
 error_setg(errp,
"RDMA is in an error state waiting migration to abort!");
 return -1;
@@ -2824,7 +2824,7 @@ static ssize_t qio_channel_rdma_writev(QIOChannel *ioc,
  */
 ret = qemu_rdma_write_flush(rdma);
 if (ret < 0) {
-rdma->error_state = ret;
+rdma->errored = true;
 error_setg(errp, "qemu_rdma_write_flush failed");
 return -1;
 }
@@ -2844,7 +2844,7 @@ static ssize_t qio_channel_rdma_writev(QIOChannel *ioc,
 ret = qemu_rdma_exchange_send(rdma, &head, data, NULL, NULL, NULL);
 
 if (ret < 0) {
-rdma->error_state = ret;
+rdma->errored = true;
 error_setg(errp, "qemu_rdma_exchange_send failed");
 return -1;
 }
@@ -2902,7 +2902,7 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
 return -1;
 }
 
-if (rdma->error_state) {
+if (rdma->errored) {
 error_setg(errp,
"RDMA is in an error state waiting migration to abort!");
 return -1;
@@ -2938,7 +2938,7 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
 ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_QEMU_FILE);
 
 if (ret < 0) {
-rdma->error_state = ret

[PULL 52/65] migration/rdma: Silence qemu_rdma_resolve_host()

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_resolve_host() violates this principle: it calls
error_report().

Clean this up: drop error_report().

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-46-arm...@redhat.com>
---
 migration/rdma.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index ee9221d5d2..3c7a407d25 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1009,7 +1009,6 @@ route:
 error_setg(errp,
"RDMA ERROR: result not equal to event_addr_resolved %s",
rdma_event_str(cm_event->event));
-error_report("rdma_resolve_addr");
 rdma_ack_cm_event(cm_event);
 goto err_resolve_get_addr;
 }
-- 
2.41.0

[PULL 01/65] migration/qmp: Fix crash on setting tls-authz with null

[Juan Quintela]

From: Peter Xu 

QEMU will crash if anyone tries to set tls-authz (which is a type
StrOrNull) with 'null' value.  Fix it in the easy way by converting it to
qstring just like the other two tls parameters.

Cc: qemu-sta...@nongnu.org # v4.0+
Fixes: d2f1d29b95 ("migration: add support for a "tls-authz" migration 
parameter")
Reviewed-by: Daniel P. Berrangé 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Philippe Mathieu-Daudé 
Reviewed-by: Juan Quintela 
Signed-off-by: Peter Xu 
Signed-off-by: Juan Quintela 
Message-ID: <20230905162335.235619-2-pet...@redhat.com>
---
 migration/options.c | 9 +++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/migration/options.c b/migration/options.c
index 1d1e1321b0..6bbfd4853d 100644
--- a/migration/options.c
+++ b/migration/options.c
@@ -1408,20 +1408,25 @@ void qmp_migrate_set_parameters(MigrateSetParameters 
*params, Error **errp)
 {
 MigrationParameters tmp;
 
-/* TODO Rewrite "" to null instead */
+/* TODO Rewrite "" to null instead for all three tls_* parameters */
 if (params->tls_creds
 && params->tls_creds->type == QTYPE_QNULL) {
 qobject_unref(params->tls_creds->u.n);
 params->tls_creds->type = QTYPE_QSTRING;
 params->tls_creds->u.s = strdup("");
 }
-/* TODO Rewrite "" to null instead */
 if (params->tls_hostname
 && params->tls_hostname->type == QTYPE_QNULL) {
 qobject_unref(params->tls_hostname->u.n);
 params->tls_hostname->type = QTYPE_QSTRING;
 params->tls_hostname->u.s = strdup("");
 }
+if (params->tls_authz
+&& params->tls_authz->type == QTYPE_QNULL) {
+qobject_unref(params->tls_authz->u.n);
+params->tls_authz->type = QTYPE_QSTRING;
+params->tls_authz->u.s = strdup("");
+}
 
 migrate_params_test_apply(params, &tmp);
 
-- 
2.41.0

[PULL 24/65] migration/rdma: Ditch useless numeric error codes in error messages

[Juan Quintela]

From: Markus Armbruster 

Several error messages include numeric error codes returned by failed
functions:

* ibv_poll_cq() returns an unspecified negative value.  Useless.

* rdma_accept and rdma_get_cm_event() return -1.  Useless.

* qemu_rdma_poll() returns either -1 or an unspecified negative
  value.  Useless.

* qemu_rdma_block_for_wrid(), qemu_rdma_write_flush(),
  qemu_rdma_exchange_send(), qemu_rdma_exchange_recv(),
  qemu_rdma_write() return a negative value that may or may not be an
  errno value.  While reporting human-readable errno
  information (which a number is not) can be useful, reporting an
  error code that may or may not be an errno value is useless.

Drop these error codes from the error messages.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-18-arm...@redhat.com>
---
 migration/rdma.c | 20 ++--
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 35b0129ae6..c4197c6437 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1476,7 +1476,7 @@ static int qemu_rdma_poll(RDMAContext *rdma, struct 
ibv_cq *cq,
 }
 
 if (ret < 0) {
-error_report("ibv_poll_cq return %d", ret);
+error_report("ibv_poll_cq failed");
 return ret;
 }
 
@@ -2226,7 +2226,7 @@ retry:
 ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
 if (ret < 0) {
 error_report("rdma migration: failed to make "
- "room in full send queue! %d", ret);
+ "room in full send queue!");
 return ret;
 }
 
@@ -2819,7 +2819,7 @@ static ssize_t qio_channel_rdma_writev(QIOChannel *ioc,
 ret = qemu_rdma_write_flush(rdma);
 if (ret < 0) {
 rdma->error_state = ret;
-error_setg(errp, "qemu_rdma_write_flush returned %d", ret);
+error_setg(errp, "qemu_rdma_write_flush failed");
 return -1;
 }
 
@@ -2839,7 +2839,7 @@ static ssize_t qio_channel_rdma_writev(QIOChannel *ioc,
 
 if (ret < 0) {
 rdma->error_state = ret;
-error_setg(errp, "qemu_rdma_exchange_send returned %d", ret);
+error_setg(errp, "qemu_rdma_exchange_send failed");
 return -1;
 }
 
@@ -2929,7 +2929,7 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
 
 if (ret < 0) {
 rdma->error_state = ret;
-error_setg(errp, "qemu_rdma_exchange_recv returned %d", ret);
+error_setg(errp, "qemu_rdma_exchange_recv failed");
 return -1;
 }
 
@@ -3271,7 +3271,7 @@ static int qemu_rdma_save_page(QEMUFile *f, ram_addr_t 
block_offset,
  */
 ret = qemu_rdma_write(rdma, block_offset, offset, size);
 if (ret < 0) {
-error_report("rdma migration: write error! %d", ret);
+error_report("rdma migration: write error");
 goto err;
 }
 
@@ -3287,7 +3287,7 @@ static int qemu_rdma_save_page(QEMUFile *f, ram_addr_t 
block_offset,
 ret = qemu_rdma_poll(rdma, rdma->recv_cq, &wr_id_in, NULL);
 
 if (ret < 0) {
-error_report("rdma migration: polling error! %d", ret);
+error_report("rdma migration: polling error");
 goto err;
 }
 
@@ -3303,7 +3303,7 @@ static int qemu_rdma_save_page(QEMUFile *f, ram_addr_t 
block_offset,
 ret = qemu_rdma_poll(rdma, rdma->send_cq, &wr_id_in, NULL);
 
 if (ret < 0) {
-error_report("rdma migration: polling error! %d", ret);
+error_report("rdma migration: polling error");
 goto err;
 }
 
@@ -3478,13 +3478,13 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 
 ret = rdma_accept(rdma->cm_id, &conn_param);
 if (ret) {
-error_report("rdma_accept returns %d", ret);
+error_report("rdma_accept failed");
 goto err_rdma_dest_wait;
 }
 
 ret = rdma_get_cm_event(rdma->channel, &cm_event);
 if (ret) {
-error_report("rdma_accept get_cm_event failed %d", ret);
+error_report("rdma_accept get_cm_event failed");
 goto err_rdma_dest_wait;
 }
 
-- 
2.41.0

[PULL 04/65] tests/qtest: migration: Use migrate_incoming_qmp where appropriate

[Juan Quintela]

From: Fabiano Rosas 

Use the new migrate_incoming_qmp helper in the places that currently
open-code calling migrate-incoming.

Reviewed-by: Juan Quintela 
Reviewed-by: Peter Xu 
Signed-off-by: Fabiano Rosas 
Signed-off-by: Juan Quintela 
Message-ID: <20230712190742.22294-4-faro...@suse.de>
---
 tests/qtest/migration-test.c  | 12 ++---
 tests/qtest/virtio-net-failover.c | 77 ---
 tests/qtest/meson.build   |  1 +
 3 files changed, 14 insertions(+), 76 deletions(-)

diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c
index 48778565a3..84660b3e3d 100644
--- a/tests/qtest/migration-test.c
+++ b/tests/qtest/migration-test.c
@@ -1970,8 +1970,7 @@ static void *test_migrate_fd_start_hook(QTestState *from,
 close(pair[0]);
 
 /* Start incoming migration from the 1st socket */
-qtest_qmp_assert_success(to, "{ 'execute': 'migrate-incoming',"
- "  'arguments': { 'uri': 'fd:fd-mig' }}");
+migrate_incoming_qmp(to, "fd:fd-mig", "{}");
 
 /* Send the 2nd socket to the target */
 qtest_qmp_fds_assert_success(from, &pair[1], 1,
@@ -2193,8 +2192,7 @@ test_migrate_precopy_tcp_multifd_start_common(QTestState 
*from,
 migrate_set_capability(to, "multifd", true);
 
 /* Start incoming migration from the 1st socket */
-qtest_qmp_assert_success(to, "{ 'execute': 'migrate-incoming',"
- "  'arguments': { 'uri': 'tcp:127.0.0.1:0' }}");
+migrate_incoming_qmp(to, "tcp:127.0.0.1:0", "{}");
 
 return NULL;
 }
@@ -2447,8 +2445,7 @@ static void test_multifd_tcp_cancel(void)
 migrate_set_capability(to, "multifd", true);
 
 /* Start incoming migration from the 1st socket */
-qtest_qmp_assert_success(to, "{ 'execute': 'migrate-incoming',"
- "  'arguments': { 'uri': 'tcp:127.0.0.1:0' }}");
+migrate_incoming_qmp(to, "tcp:127.0.0.1:0", "{}");
 
 /* Wait for the first serial output from the source */
 wait_for_serial("src_serial");
@@ -2478,8 +2475,7 @@ static void test_multifd_tcp_cancel(void)
 migrate_set_capability(to2, "multifd", true);
 
 /* Start incoming migration from the 1st socket */
-qtest_qmp_assert_success(to2, "{ 'execute': 'migrate-incoming',"
- "  'arguments': { 'uri': 'tcp:127.0.0.1:0' }}");
+migrate_incoming_qmp(to2, "tcp:127.0.0.1:0", "{}");
 
 g_free(uri);
 uri = migrate_get_socket_address(to2, "socket-address");
diff --git a/tests/qtest/virtio-net-failover.c 
b/tests/qtest/virtio-net-failover.c
index 4a809590bf..0d40bc1f2d 100644
--- a/tests/qtest/virtio-net-failover.c
+++ b/tests/qtest/virtio-net-failover.c
@@ -11,6 +11,7 @@
 #include "libqtest.h"
 #include "libqos/pci.h"
 #include "libqos/pci-pc.h"
+#include "migration-helpers.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qlist.h"
 #include "qapi/qmp/qjson.h"
@@ -736,26 +737,10 @@ static void test_migrate_out(gconstpointer opaque)
 machine_stop(qts);
 }
 
-static QDict *get_migration_event(QTestState *qts)
-{
-QDict *resp;
-QDict *data;
-
-resp = qtest_qmp_eventwait_ref(qts, "MIGRATION");
-g_assert(qdict_haskey(resp, "data"));
-
-data = qdict_get_qdict(resp, "data");
-g_assert(qdict_haskey(data, "status"));
-qobject_ref(data);
-qobject_unref(resp);
-
-return data;
-}
-
 static void test_migrate_in(gconstpointer opaque)
 {
 QTestState *qts;
-QDict *resp, *args, *ret;
+QDict *resp, *ret;
 g_autofree gchar *uri = g_strdup_printf("exec: cat %s", (gchar *)opaque);
 
 qts = machine_start(BASE_MACHINE
@@ -787,18 +772,7 @@ static void test_migrate_in(gconstpointer opaque)
 check_one_card(qts, true, "standby0", MAC_STANDBY0);
 check_one_card(qts, false, "primary0", MAC_PRIMARY0);
 
-args = qdict_from_jsonf_nofail("{}");
-g_assert_nonnull(args);
-qdict_put_str(args, "uri", uri);
-
-resp = qtest_qmp(qts, "{ 'execute': 'migrate-incoming', 'arguments': %p}",
- args);
-g_assert(qdict_haskey(resp, "return"));
-qobject_unref(resp);
-
-resp = get_migration_event(qts);
-g_assert_cmpstr(qdict_get_str(resp, "status"), ==, "setup");
-qobject_unref(resp);
+migrate_incoming_qmp(qts, uri, "{}");
 
 resp = get_failover_negociated_event(qts);
 g_assert_cmpstr(qdict_get_str(resp, "device-id"), ==, "standby0");
@@ -888,7 +862,7 @@ static void test_off_migrate_out(gconstpointer opaque)
 static void test_off_migrate_in(gconstpointer opaque)
 {
 QTestState *qts;
-QDict *resp, *args, *ret;
+QDict *ret;
 g_autofree gchar *uri = g_strdup_printf("exec: cat %s", (gchar *)opaque);
 
 qts = machine_start(BASE_MACHINE
@@ -920,18 +894,7 @@ static void test_off_migrate_in(gconstpointer opaque)
 check_one_card(qts, true, "standby0", MAC_STANDBY0);
 check_one_card(qts, true, "primary0", MAC_PRIMARY0);
 
-args = qdict_from_jsonf_nofail("{}");
-g_assert_nonnull(args);
-qdict_put_s

[PULL 36/65] migration/rdma: Check negative error values the same way everywhere

[Juan Quintela]

From: Markus Armbruster 

When a function returns 0 on success, negative value on error,
checking for non-zero suffices, but checking for negative is clearer.
So do that.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-30-arm...@redhat.com>
---
 migration/rdma.c | 82 
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 08a82d5e57..b0125b01cf 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -953,7 +953,7 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error 
**errp)
 
 /* create CM id */
 ret = rdma_create_id(rdma->channel, &rdma->cm_id, NULL, RDMA_PS_TCP);
-if (ret) {
+if (ret < 0) {
 ERROR(errp, "could not create channel id");
 goto err_resolve_create_id;
 }
@@ -974,10 +974,10 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, 
Error **errp)
 
 ret = rdma_resolve_addr(rdma->cm_id, NULL, e->ai_dst_addr,
 RDMA_RESOLVE_TIMEOUT_MS);
-if (!ret) {
+if (ret >= 0) {
 if (e->ai_family == AF_INET6) {
 ret = qemu_rdma_broken_ipv6_kernel(rdma->cm_id->verbs, errp);
-if (ret) {
+if (ret < 0) {
 continue;
 }
 }
@@ -994,7 +994,7 @@ route:
 qemu_rdma_dump_gid("source_resolve_addr", rdma->cm_id);
 
 ret = rdma_get_cm_event(rdma->channel, &cm_event);
-if (ret) {
+if (ret < 0) {
 ERROR(errp, "could not perform event_addr_resolved");
 goto err_resolve_get_addr;
 }
@@ -1010,13 +1010,13 @@ route:
 
 /* resolve route */
 ret = rdma_resolve_route(rdma->cm_id, RDMA_RESOLVE_TIMEOUT_MS);
-if (ret) {
+if (ret < 0) {
 ERROR(errp, "could not resolve rdma route");
 goto err_resolve_get_addr;
 }
 
 ret = rdma_get_cm_event(rdma->channel, &cm_event);
-if (ret) {
+if (ret < 0) {
 ERROR(errp, "could not perform event_route_resolved");
 goto err_resolve_get_addr;
 }
@@ -1124,7 +1124,7 @@ static int qemu_rdma_alloc_qp(RDMAContext *rdma)
 attr.qp_type = IBV_QPT_RC;
 
 ret = rdma_create_qp(rdma->cm_id, rdma->pd, &attr);
-if (ret) {
+if (ret < 0) {
 return -1;
 }
 
@@ -1567,7 +1567,7 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma,
 
 if (pfds[1].revents) {
 ret = rdma_get_cm_event(rdma->channel, &cm_event);
-if (ret) {
+if (ret < 0) {
 error_report("failed to get cm event while wait "
  "completion channel");
 return -1;
@@ -1668,12 +1668,12 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma,
 
 while (1) {
 ret = qemu_rdma_wait_comp_channel(rdma, ch);
-if (ret) {
+if (ret < 0) {
 goto err_block_for_wrid;
 }
 
 ret = ibv_get_cq_event(ch, &cq, &cq_ctx);
-if (ret) {
+if (ret < 0) {
 /*
  * FIXME perror() is problematic, because ibv_reg_mr() is
  * not documented to set errno.  Will go away later in
@@ -1911,7 +1911,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
  */
 if (resp) {
 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA);
-if (ret) {
+if (ret < 0) {
 error_report("rdma migration: error posting"
 " extra control recv for anticipated result!");
 return -1;
@@ -1922,7 +1922,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
  * Post a WR to replace the one we just consumed for the READY message.
  */
 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
-if (ret) {
+if (ret < 0) {
 error_report("rdma migration: error posting first control recv!");
 return -1;
 }
@@ -2009,7 +2009,7 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
  * Post a new RECV work request to replace the one we just consumed.
  */
 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
-if (ret) {
+if (ret < 0) {
 error_report("rdma migration: error posting second control recv!");
 return -1;
 }
@@ -2357,7 +2357,7 @@ static int qemu_rdma_write(RDMAContext *rdma,
 /* If we cannot merge it, we flush the current buffer first. */
 if (!qemu_rdma_buffer_mergeable(rdma, current_addr, len)) {
 ret = qemu_rdma_write_flush(rdma);
-if (ret) {
+if (ret < 0) {
 return -1;
 }
 rdma->current_length = 0;
@@ -2487,12 +2487,12 @@ static int qemu_rdma_source_init(RDMAContext *rdma, 
bool pin_all, Error **errp)
 rdma->pin_all = pin_all;
 

[PULL 38/65] migration/rdma: Delete inappropriate error_report() in macro ERROR()

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

Macro ERROR() violates this principle.  Delete the error_report()
there.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Tested-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-32-arm...@redhat.com>
---
 migration/rdma.c | 4 
 1 file changed, 4 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 00e3c430f4..6c0e6cda2c 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -40,12 +40,8 @@
 #include "options.h"
 #include 
 
-/*
- * Print and error on both the Monitor and the Log file.
- */
 #define ERROR(errp, fmt, ...) \
 do { \
-fprintf(stderr, "RDMA ERROR: " fmt "\n", ## __VA_ARGS__); \
 if (errp && (*(errp) == NULL)) { \
 error_setg(errp, "RDMA ERROR: " fmt, ## __VA_ARGS__); \
 } \
-- 
2.41.0

[PULL 65/65] migration: Add migration_rp_wait|kick()

[Juan Quintela]

From: Peter Xu 

It's just a simple wrapper for rp_sem on either wait() or kick(), make it
even clearer on how it is used.  Prepared to be used even for other things.

Reviewed-by: Fabiano Rosas 
Signed-off-by: Peter Xu 
Message-ID: <20231004220240.167175-8-pet...@redhat.com>
Signed-off-by: Juan Quintela 
---
 migration/migration.h | 15 +++
 migration/migration.c | 14 --
 migration/ram.c   | 16 +++-
 3 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/migration/migration.h b/migration/migration.h
index 4106a1dc54..cd5534337c 100644
--- a/migration/migration.h
+++ b/migration/migration.h
@@ -316,6 +316,12 @@ struct MigrationState {
  * be cleared in the rp_thread!
  */
 bool  rp_thread_created;
+/*
+ * Used to synchronize between migration main thread and return
+ * path thread.  The migration thread can wait() on this sem, while
+ * other threads (e.g., return path thread) can kick it using a
+ * post().
+ */
 QemuSemaphore rp_sem;
 /*
  * We post to this when we got one PONG from dest. So far it's an
@@ -527,4 +533,13 @@ void migration_populate_vfio_info(MigrationInfo *info);
 void migration_reset_vfio_bytes_transferred(void);
 void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page);
 
+/* Migration thread waiting for return path thread. */
+void migration_rp_wait(MigrationState *s);
+/*
+ * Kick the migration thread waiting for return path messages.  NOTE: the
+ * name can be slightly confusing (when read as "kick the rp thread"), just
+ * to remember the target is always the migration thread.
+ */
+void migration_rp_kick(MigrationState *s);
+
 #endif
diff --git a/migration/migration.c b/migration/migration.c
index 409eb3e916..1c6c81ad49 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -1763,6 +1763,16 @@ static void mark_source_rp_bad(MigrationState *s)
 s->rp_state.error = true;
 }
 
+void migration_rp_wait(MigrationState *s)
+{
+qemu_sem_wait(&s->rp_state.rp_sem);
+}
+
+void migration_rp_kick(MigrationState *s)
+{
+qemu_sem_post(&s->rp_state.rp_sem);
+}
+
 static struct rp_cmd_args {
 ssize_t len; /* -1 = variable */
 const char *name;
@@ -1835,7 +1845,7 @@ static int migrate_handle_rp_resume_ack(MigrationState 
*s, uint32_t value)
   MIGRATION_STATUS_POSTCOPY_ACTIVE);
 
 /* Notify send thread that time to continue send pages */
-qemu_sem_post(&s->rp_state.rp_sem);
+migration_rp_kick(s);
 
 return 0;
 }
@@ -2464,7 +2474,7 @@ static int postcopy_resume_handshake(MigrationState *s)
 qemu_savevm_send_postcopy_resume(s->to_dst_file);
 
 while (s->state == MIGRATION_STATUS_POSTCOPY_RECOVER) {
-qemu_sem_wait(&s->rp_state.rp_sem);
+migration_rp_wait(s);
 }
 
 if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
diff --git a/migration/ram.c b/migration/ram.c
index 6c40d9af0c..2f5ce4d60b 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -4141,7 +4141,7 @@ static int ram_dirty_bitmap_sync_all(MigrationState *s, 
RAMState *rs)
 
 /* Wait until all the ramblocks' dirty bitmap synced */
 while (qatomic_read(&rs->postcopy_bmap_sync_requested)) {
-qemu_sem_wait(&s->rp_state.rp_sem);
+migration_rp_wait(s);
 }
 
 trace_ram_dirty_bitmap_sync_complete();
@@ -4149,11 +4149,6 @@ static int ram_dirty_bitmap_sync_all(MigrationState *s, 
RAMState *rs)
 return 0;
 }
 
-static void ram_dirty_bitmap_reload_notify(MigrationState *s)
-{
-qemu_sem_post(&s->rp_state.rp_sem);
-}
-
 /*
  * Read the received bitmap, revert it as the initial dirty bitmap.
  * This is only used when the postcopy migration is paused but wants
@@ -4237,10 +4232,13 @@ int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock 
*block)
 qatomic_dec(&rs->postcopy_bmap_sync_requested);
 
 /*
- * We succeeded to sync bitmap for current ramblock. If this is
- * the last one to sync, we need to notify the main send thread.
+ * We succeeded to sync bitmap for current ramblock. Always kick the
+ * migration thread to check whether all requested bitmaps are
+ * reloaded.  NOTE: it's racy to only kick when requested==0, because
+ * we don't know whether the migration thread may still be increasing
+ * it.
  */
-ram_dirty_bitmap_reload_notify(s);
+migration_rp_kick(s);
 
 ret = 0;
 out:
-- 
2.41.0

[PULL 14/65] migration/rdma: Clean up two more harmless signed vs. unsigned issues

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_exchange_get_response() compares int parameter @expecting
with uint32_t head->type.  Actual arguments are non-negative
enumeration constants, RDMAControlHeader uint32_t member type, or
qemu_rdma_exchange_recv() int parameter expecting.  Actual arguments
for the latter are non-negative enumeration constants.  Change both
parameters to uint32_t.

In qio_channel_rdma_readv(), loop control variable @i is ssize_t, and
counts from 0 up to @niov, which is size_t.  Change @i to size_t.

While there, make qio_channel_rdma_readv() and
qio_channel_rdma_writev() more consistent: change the former's @done
to ssize_t, and delete the latter's useless initialization of @len.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-8-arm...@redhat.com>
---
 migration/rdma.c | 11 +--
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index ff8e475f59..1aac50b0de 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1801,7 +1801,7 @@ static int qemu_rdma_post_recv_control(RDMAContext *rdma, 
int idx)
  * Block and wait for a RECV control channel message to arrive.
  */
 static int qemu_rdma_exchange_get_response(RDMAContext *rdma,
-RDMAControlHeader *head, int expecting, int idx)
+RDMAControlHeader *head, uint32_t expecting, int idx)
 {
 uint32_t byte_len;
 int ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RECV_CONTROL + idx,
@@ -1961,7 +1961,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
  * control-channel message.
  */
 static int qemu_rdma_exchange_recv(RDMAContext *rdma, RDMAControlHeader *head,
-int expecting)
+   uint32_t expecting)
 {
 RDMAControlHeader ready = {
 .len = 0,
@@ -2784,8 +2784,7 @@ static ssize_t qio_channel_rdma_writev(QIOChannel *ioc,
 RDMAContext *rdma;
 int ret;
 ssize_t done = 0;
-size_t i;
-size_t len = 0;
+size_t i, len;
 
 RCU_READ_LOCK_GUARD();
 rdma = qatomic_rcu_read(&rioc->rdmaout);
@@ -2870,8 +2869,8 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
 RDMAContext *rdma;
 RDMAControlHeader head;
 int ret = 0;
-ssize_t i;
-size_t done = 0, len;
+ssize_t done = 0;
+size_t i, len;
 
 RCU_READ_LOCK_GUARD();
 rdma = qatomic_rcu_read(&rioc->rdmain);
-- 
2.41.0

[PULL 08/65] migration/rdma: Clean up qemu_rdma_poll()'s return type

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_poll()'s return type is uint64_t, even though it returns 0,
-1, or @ret, which is int.  Its callers assign the return value to int
variables, then check whether it's negative.  Unclean.

Return int instead.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-2-arm...@redhat.com>
---
 migration/rdma.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index cd5e1afe60..e72864d1cf 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1469,8 +1469,8 @@ static uint64_t qemu_rdma_make_wrid(uint64_t wr_id, 
uint64_t index,
  * (of any kind) has completed.
  * Return the work request ID that completed.
  */
-static uint64_t qemu_rdma_poll(RDMAContext *rdma, struct ibv_cq *cq,
-   uint64_t *wr_id_out, uint32_t *byte_len)
+static int qemu_rdma_poll(RDMAContext *rdma, struct ibv_cq *cq,
+  uint64_t *wr_id_out, uint32_t *byte_len)
 {
 int ret;
 struct ibv_wc wc;
-- 
2.41.0

[PULL 23/65] migration/rdma: Fix or document problematic uses of errno

[Juan Quintela]

From: Markus Armbruster 

We use errno after calling Libibverbs functions that are not
documented to set errno (manual page does not mention errno), or where
the documentation is unclear ("returns [...] the value of errno on
failure").  While this could be read as "sets errno and returns it",
a glance at the source code[*] kills that hope:

static inline int ibv_post_send(struct ibv_qp *qp, struct ibv_send_wr *wr,
struct ibv_send_wr **bad_wr)
{
return qp->context->ops.post_send(qp, wr, bad_wr);
}

The callback can be

static int mana_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
  struct ibv_send_wr **bad)
{
/* This version of driver supports RAW QP only.
 * Posting WR is done directly in the application.
 */
return EOPNOTSUPP;
}

Neither of them touches errno.

One of these errno uses is easy to fix, so do that now.  Several more
will go away later in the series; add temporary FIXME commments.
Three will remain; add TODO comments.  TODO, not FIXME, because the
bug might be in Libibverbs documentation.

[*] https://github.com/linux-rdma/rdma-core.git
commit 55fa316b4b18f258d8ac1ceb4aa5a7a35b094dcf

Signed-off-by: Markus Armbruster 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-17-arm...@redhat.com>
---
 migration/rdma.c | 45 +++--
 1 file changed, 39 insertions(+), 6 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index dffca30382..35b0129ae6 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -853,6 +853,12 @@ static int qemu_rdma_broken_ipv6_kernel(struct ibv_context 
*verbs, Error **errp)
 
 for (x = 0; x < num_devices; x++) {
 verbs = ibv_open_device(dev_list[x]);
+/*
+ * ibv_open_device() is not documented to set errno.  If
+ * it does, it's somebody else's doc bug.  If it doesn't,
+ * the use of errno below is wrong.
+ * TODO Find out whether ibv_open_device() sets errno.
+ */
 if (!verbs) {
 if (errno == EPERM) {
 continue;
@@ -1162,11 +1168,7 @@ static void qemu_rdma_advise_prefetch_mr(struct ibv_pd 
*pd, uint64_t addr,
 ret = ibv_advise_mr(pd, advice,
 IBV_ADVISE_MR_FLAG_FLUSH, &sg_list, 1);
 /* ignore the error */
-if (ret) {
-trace_qemu_rdma_advise_mr(name, len, addr, strerror(errno));
-} else {
-trace_qemu_rdma_advise_mr(name, len, addr, "successed");
-}
+trace_qemu_rdma_advise_mr(name, len, addr, strerror(ret));
 #endif
 }
 
@@ -1183,7 +1185,12 @@ static int qemu_rdma_reg_whole_ram_blocks(RDMAContext 
*rdma)
 local->block[i].local_host_addr,
 local->block[i].length, access
 );
-
+/*
+ * ibv_reg_mr() is not documented to set errno.  If it does,
+ * it's somebody else's doc bug.  If it doesn't, the use of
+ * errno below is wrong.
+ * TODO Find out whether ibv_reg_mr() sets errno.
+ */
 if (!local->block[i].mr &&
 errno == ENOTSUP && rdma_support_odp(rdma->verbs)) {
 access |= IBV_ACCESS_ON_DEMAND;
@@ -1291,6 +1298,12 @@ static int qemu_rdma_register_and_get_keys(RDMAContext 
*rdma,
 trace_qemu_rdma_register_and_get_keys(len, chunk_start);
 
 block->pmr[chunk] = ibv_reg_mr(rdma->pd, chunk_start, len, access);
+/*
+ * ibv_reg_mr() is not documented to set errno.  If it does,
+ * it's somebody else's doc bug.  If it doesn't, the use of
+ * errno below is wrong.
+ * TODO Find out whether ibv_reg_mr() sets errno.
+ */
 if (!block->pmr[chunk] &&
 errno == ENOTSUP && rdma_support_odp(rdma->verbs)) {
 access |= IBV_ACCESS_ON_DEMAND;
@@ -1408,6 +1421,11 @@ static int qemu_rdma_unregister_waiting(RDMAContext 
*rdma)
 block->remote_keys[chunk] = 0;
 
 if (ret != 0) {
+/*
+ * FIXME perror() is problematic, bcause ibv_dereg_mr() is
+ * not documented to set errno.  Will go away later in
+ * this series.
+ */
 perror("unregistration chunk failed");
 return -ret;
 }
@@ -1658,6 +1676,11 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma,
 
 ret = ibv_get_cq_event(ch, &cq, &cq_ctx);
 if (ret) {
+/*
+ * FIXME perror() is problematic, because ibv_reg_mr() is
+ * not documented to set errno.  Will go away later in
+ * this series.
+ */
 perror("ibv_get_cq_event");
 goto err_block_for_wrid;
 }
@@ -2210,6 +2233,11 @@ retry:
 goto retry;
 
 } else if (ret > 0) {
+/*
+ * FIX

[PULL 07/65] migration: Allow RECOVER->PAUSED convertion for dest qemu

[Juan Quintela]

From: Peter Xu 

There's a bug on dest that if a double fault triggered on dest qemu (a
network issue during postcopy-recover), we won't set PAUSED correctly
because we assumed we always came from ACTIVE.

Fix that by always overwriting the state to PAUSE.

We could also check for these two states, but maybe it's an overkill.  We
did the same on the src QEMU to unconditionally switch to PAUSE anyway.

Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Peter Xu 
Signed-off-by: Juan Quintela 
Message-ID: <20231004220240.167175-10-pet...@redhat.com>
---
 migration/savevm.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/migration/savevm.c b/migration/savevm.c
index 60eec7c31f..497ce02bd7 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -2734,7 +2734,8 @@ static bool 
postcopy_pause_incoming(MigrationIncomingState *mis)
 qemu_mutex_unlock(&mis->postcopy_prio_thread_mutex);
 }
 
-migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+/* Current state can be either ACTIVE or RECOVER */
+migrate_set_state(&mis->state, mis->state,
   MIGRATION_STATUS_POSTCOPY_PAUSED);
 
 /* Notify the fault thread for the invalidated file handle */
-- 
2.41.0

[PULL 47/65] migration/rdma: Convert qemu_rdma_write_one() to Error

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_write_flush() violates this principle: it calls
error_report() via qemu_rdma_write_one().  I elected not to
investigate how callers handle the error, i.e. precise impact is not
known.

Clean this up by converting qemu_rdma_write_one() to Error.  Bonus:
resolves a FIXME about problematic use of errno.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-41-arm...@redhat.com>
---
 migration/rdma.c | 32 
 1 file changed, 12 insertions(+), 20 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 1a74c6d955..369d30c895 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2041,9 +2041,8 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
  */
 static int qemu_rdma_write_one(RDMAContext *rdma,
int current_index, uint64_t current_addr,
-   uint64_t length)
+   uint64_t length, Error **errp)
 {
-Error *err = NULL;
 struct ibv_sge sge;
 struct ibv_send_wr send_wr = { 0 };
 struct ibv_send_wr *bad_wr;
@@ -2097,7 +2096,7 @@ retry:
 ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
 
 if (ret < 0) {
-error_report("Failed to Wait for previous write to complete "
+error_setg(errp, "Failed to Wait for previous write to complete "
 "block %d chunk %" PRIu64
 " current %" PRIu64 " len %" PRIu64 " %d",
 current_index, chunk, sge.addr, length, rdma->nb_sent);
@@ -2129,10 +2128,9 @@ retry:
 
 compress_to_network(rdma, &comp);
 ret = qemu_rdma_exchange_send(rdma, &head,
-(uint8_t *) &comp, NULL, NULL, NULL, &err);
+(uint8_t *) &comp, NULL, NULL, NULL, errp);
 
 if (ret < 0) {
-error_report_err(err);
 return -1;
 }
 
@@ -2167,9 +2165,8 @@ retry:
 
 register_to_network(rdma, ®);
 ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) ®,
-&resp, ®_result_idx, NULL, &err);
+&resp, ®_result_idx, NULL, errp);
 if (ret < 0) {
-error_report_err(err);
 return -1;
 }
 
@@ -2177,7 +2174,7 @@ retry:
 if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr,
 &sge.lkey, NULL, chunk,
 chunk_start, chunk_end)) {
-error_report("cannot get lkey");
+error_setg(errp, "cannot get lkey");
 return -1;
 }
 
@@ -2196,7 +2193,7 @@ retry:
 if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr,
 &sge.lkey, NULL, chunk,
 chunk_start, chunk_end)) {
-error_report("cannot get lkey!");
+error_setg(errp, "cannot get lkey!");
 return -1;
 }
 }
@@ -2208,7 +2205,7 @@ retry:
 if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr,
  &sge.lkey, NULL, chunk,
  chunk_start, chunk_end)) {
-error_report("cannot get lkey!");
+error_setg(errp, "cannot get lkey!");
 return -1;
 }
 }
@@ -2242,7 +2239,7 @@ retry:
 trace_qemu_rdma_write_one_queue_full();
 ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
 if (ret < 0) {
-error_report("rdma migration: failed to make "
+error_setg(errp, "rdma migration: failed to make "
  "room in full send queue!");
 return -1;
 }
@@ -2250,12 +2247,8 @@ retry:
 goto retry;
 
 } else if (ret > 0) {
-/*
- * FIXME perror() is problematic, because whether
- * ibv_post_send() sets errno is unclear.  Will go away later
- * in this series.
- */
-perror("rdma migration: post rdma write failed");
+error_setg_errno(errp, ret,
+ "rdma migration: post rdma write failed");
 return -1;
 }
 
@@ -2291,11 +2284,10 @@ static int qemu_rdma_write_flush(RDMAContext *rdma, 
Error **errp)
 return 0;
   

[PULL 35/65] migration/rdma: Drop superfluous assignments to @ret

[Juan Quintela]

From: Markus Armbruster 

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-29-arm...@redhat.com>
---
 migration/rdma.c | 35 ++-
 1 file changed, 10 insertions(+), 25 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index a57ec3791a..08a82d5e57 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1530,7 +1530,7 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma,
struct ibv_comp_channel *comp_channel)
 {
 struct rdma_cm_event *cm_event;
-int ret = -1;
+int ret;
 
 /*
  * Coroutine doesn't start until migration_fd_process_incoming()
@@ -1635,7 +1635,7 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma,
 uint64_t wrid_requested,
 uint32_t *byte_len)
 {
-int num_cq_events = 0, ret = 0;
+int num_cq_events = 0, ret;
 struct ibv_cq *cq;
 void *cq_ctx;
 uint64_t wr_id = RDMA_WRID_NONE, wr_id_in;
@@ -1685,8 +1685,7 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma,
 
 num_cq_events++;
 
-ret = -ibv_req_notify_cq(cq, 0);
-if (ret) {
+if (ibv_req_notify_cq(cq, 0)) {
 goto err_block_for_wrid;
 }
 
@@ -1733,7 +1732,7 @@ err_block_for_wrid:
 static int qemu_rdma_post_send_control(RDMAContext *rdma, uint8_t *buf,
RDMAControlHeader *head)
 {
-int ret = 0;
+int ret;
 RDMAWorkRequestData *wr = &rdma->wr_data[RDMA_WRID_CONTROL];
 struct ibv_send_wr *bad_wr;
 struct ibv_sge sge = {
@@ -1890,7 +1889,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
int *resp_idx,
int (*callback)(RDMAContext *rdma))
 {
-int ret = 0;
+int ret;
 
 /*
  * Wait until the dest is ready before attempting to deliver the message
@@ -2890,7 +2889,7 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
 QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc);
 RDMAContext *rdma;
 RDMAControlHeader head;
-int ret = 0;
+int ret;
 ssize_t done = 0;
 size_t i, len;
 
@@ -3379,7 +3378,7 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 RDMAContext *rdma_return_path = NULL;
 struct rdma_cm_event *cm_event;
 struct ibv_context *verbs;
-int ret = -EINVAL;
+int ret;
 int idx;
 
 ret = rdma_get_cm_event(rdma->channel, &cm_event);
@@ -3389,7 +3388,6 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 
 if (cm_event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
 rdma_ack_cm_event(cm_event);
-ret = -1;
 goto err_rdma_dest_wait;
 }
 
@@ -3402,7 +3400,6 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 rdma_return_path = qemu_rdma_data_init(rdma->host_port, NULL);
 if (rdma_return_path == NULL) {
 rdma_ack_cm_event(cm_event);
-ret = -1;
 goto err_rdma_dest_wait;
 }
 
@@ -3417,7 +3414,6 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 error_report("Unknown source RDMA version: %d, bailing...",
  cap.version);
 rdma_ack_cm_event(cm_event);
-ret = -1;
 goto err_rdma_dest_wait;
 }
 
@@ -3450,7 +3446,6 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 } else if (rdma->verbs != verbs) {
 error_report("ibv context not matching %p, %p!", rdma->verbs,
  verbs);
-ret = -1;
 goto err_rdma_dest_wait;
 }
 
@@ -3504,7 +3499,6 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) {
 error_report("rdma_accept not event established");
 rdma_ack_cm_event(cm_event);
-ret = -1;
 goto err_rdma_dest_wait;
 }
 
@@ -3567,7 +3561,7 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 static RDMARegisterResult results[RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE];
 RDMALocalBlock *block;
 void *host_addr;
-int ret = 0;
+int ret;
 int idx = 0;
 int count = 0;
 int i = 0;
@@ -3596,7 +3590,6 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 if (head.repeat > RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE) {
 error_report("rdma: Too many requests in this message (%d)."
 "Bailing.", head.repeat);
-ret = -EIO;
 break;
 }
 
@@ -3612,7 +3605,6 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 error_report("rdma: 'compress' bad block index %u (vs %d)",
  (unsigned int)comp->block_idx,
  rdma->local_ram_blocks.nb_blocks);
-ret = -EIO;
 goto err;
 }
 block = &(rdma->local_ram_blocks.

[PULL 43/65] migration/rdma: Convert qemu_rdma_exchange_send() to Error

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qio_channel_rdma_writev() violates this principle: it calls
error_report() via qemu_rdma_exchange_send().  I elected not to
investigate how callers handle the error, i.e. precise impact is not
known.

Clean this up by converting qemu_rdma_exchange_send() to Error.

Necessitates setting an error when qemu_rdma_post_recv_control(),
callback(), or qemu_rdma_exchange_get_response() failed.  Since these
errors will go away later in this series, simply use "FIXME temporary
error message" there.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-37-arm...@redhat.com>
---
 migration/rdma.c | 40 +++-
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index b13fec6328..7866cf9bb7 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -518,7 +518,8 @@ static void network_to_result(RDMARegisterResult *result)
 static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
uint8_t *data, RDMAControlHeader *resp,
int *resp_idx,
-   int (*callback)(RDMAContext *rdma));
+   int (*callback)(RDMAContext *rdma),
+   Error **errp);
 
 static inline uint64_t ram_chunk_index(const uint8_t *start,
const uint8_t *host)
@@ -1376,6 +1377,8 @@ static int qemu_rdma_reg_control(RDMAContext *rdma, int 
idx)
  */
 static int qemu_rdma_unregister_waiting(RDMAContext *rdma)
 {
+Error *err = NULL;
+
 while (rdma->unregistrations[rdma->unregister_current]) {
 int ret;
 uint64_t wr_id = rdma->unregistrations[rdma->unregister_current];
@@ -1438,8 +1441,9 @@ static int qemu_rdma_unregister_waiting(RDMAContext *rdma)
 reg.key.chunk = chunk;
 register_to_network(rdma, ®);
 ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) ®,
-&resp, NULL, NULL);
+  &resp, NULL, NULL, &err);
 if (ret < 0) {
+error_report_err(err);
 return -1;
 }
 
@@ -1893,7 +1897,8 @@ static void qemu_rdma_move_header(RDMAContext *rdma, int 
idx,
 static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
uint8_t *data, RDMAControlHeader *resp,
int *resp_idx,
-   int (*callback)(RDMAContext *rdma))
+   int (*callback)(RDMAContext *rdma),
+   Error **errp)
 {
 int ret;
 
@@ -1908,6 +1913,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
   RDMA_CONTROL_READY,
   RDMA_WRID_READY);
 if (ret < 0) {
+error_setg(errp, "FIXME temporary error message");
 return -1;
 }
 }
@@ -1918,7 +1924,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 if (resp) {
 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA);
 if (ret < 0) {
-error_report("rdma migration: error posting"
+error_setg(errp, "rdma migration: error posting"
 " extra control recv for anticipated result!");
 return -1;
 }
@@ -1929,7 +1935,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
  */
 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
 if (ret < 0) {
-error_report("rdma migration: error posting first control recv!");
+error_setg(errp, "rdma migration: error posting first control recv!");
 return -1;
 }
 
@@ -1939,7 +1945,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 ret = qemu_rdma_post_send_control(rdma, data, head);
 
 if (ret < 0) {
-error_report("Failed to send control buffer!");
+error_setg(errp, "Failed to send control buffer!");
 return -1;
 }
 
@@ -1951,6 +1957,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 trace_qemu_rdma_exchange_send_issue_callback();
 ret = callback(rdma);
 if (ret < 0) {
+error_setg(errp, "FIXME temporary error message");
 return -1;
 }
 }
@@ -1960,6 +19

[PULL 29/65] migration/rdma: Drop dead qemu_rdma_data_init() code for !@host_port

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_data_init() neglects to set an Error when it fails because
@host_port is null.  Fortunately, no caller passes null, so this is
merely a latent bug.  Drop the flawed code handling null argument.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-23-arm...@redhat.com>
---
 migration/rdma.c | 29 +
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 0039295a15..55eb8222ea 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2767,25 +2767,22 @@ static RDMAContext *qemu_rdma_data_init(const char 
*host_port, Error **errp)
 RDMAContext *rdma = NULL;
 InetSocketAddress *addr;
 
-if (host_port) {
-rdma = g_new0(RDMAContext, 1);
-rdma->current_index = -1;
-rdma->current_chunk = -1;
+rdma = g_new0(RDMAContext, 1);
+rdma->current_index = -1;
+rdma->current_chunk = -1;
 
-addr = g_new(InetSocketAddress, 1);
-if (!inet_parse(addr, host_port, NULL)) {
-rdma->port = atoi(addr->port);
-rdma->host = g_strdup(addr->host);
-rdma->host_port = g_strdup(host_port);
-} else {
-ERROR(errp, "bad RDMA migration address '%s'", host_port);
-g_free(rdma);
-rdma = NULL;
-}
-
-qapi_free_InetSocketAddress(addr);
+addr = g_new(InetSocketAddress, 1);
+if (!inet_parse(addr, host_port, NULL)) {
+rdma->port = atoi(addr->port);
+rdma->host = g_strdup(addr->host);
+rdma->host_port = g_strdup(host_port);
+} else {
+ERROR(errp, "bad RDMA migration address '%s'", host_port);
+g_free(rdma);
+rdma = NULL;
 }
 
+qapi_free_InetSocketAddress(addr);
 return rdma;
 }
 
-- 
2.41.0

[PULL 41/65] migration/rdma: Drop "@errp is clear" guards around error_setg()

[Juan Quintela]

From: Markus Armbruster 

These guards are all redundant now.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-35-arm...@redhat.com>
---
 migration/rdma.c | 164 +++
 1 file changed, 51 insertions(+), 113 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 3fb899f963..fdb527af39 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -858,10 +858,8 @@ static int qemu_rdma_broken_ipv6_kernel(struct ibv_context 
*verbs, Error **errp)
 
 if (ibv_query_port(verbs, 1, &port_attr)) {
 ibv_close_device(verbs);
-if (errp && !*errp) {
-error_setg(errp,
-   "RDMA ERROR: Could not query initial IB port");
-}
+error_setg(errp,
+   "RDMA ERROR: Could not query initial IB port");
 return -1;
 }
 
@@ -884,12 +882,10 @@ static int qemu_rdma_broken_ipv6_kernel(struct 
ibv_context *verbs, Error **errp)
 " migrate over the IB fabric until the kernel "
 " fixes the bug.\n");
 } else {
-if (errp && !*errp) {
-error_setg(errp, "RDMA ERROR: "
-   "You only have RoCE / iWARP devices in your 
systems"
-   " and your management software has specified 
'[::]'"
-   ", but IPv6 over RoCE / iWARP is not supported 
in Linux.");
-}
+error_setg(errp, "RDMA ERROR: "
+   "You only have RoCE / iWARP devices in your systems"
+   " and your management software has specified '[::]'"
+   ", but IPv6 over RoCE / iWARP is not supported in 
Linux.");
 return -1;
 }
 }
@@ -905,18 +901,14 @@ static int qemu_rdma_broken_ipv6_kernel(struct 
ibv_context *verbs, Error **errp)
 
 /* IB ports start with 1, not 0 */
 if (ibv_query_port(verbs, 1, &port_attr)) {
-if (errp && !*errp) {
-error_setg(errp, "RDMA ERROR: Could not query initial IB port");
-}
+error_setg(errp, "RDMA ERROR: Could not query initial IB port");
 return -1;
 }
 
 if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {
-if (errp && !*errp) {
-error_setg(errp, "RDMA ERROR: "
-   "Linux kernel's RoCE / iWARP does not support IPv6 "
-   "(but patches on linux-rdma in progress)");
-}
+error_setg(errp, "RDMA ERROR: "
+   "Linux kernel's RoCE / iWARP does not support IPv6 "
+   "(but patches on linux-rdma in progress)");
 return -1;
 }
 
@@ -941,27 +933,21 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, 
Error **errp)
 struct rdma_addrinfo *e;
 
 if (rdma->host == NULL || !strcmp(rdma->host, "")) {
-if (errp && !*errp) {
-error_setg(errp, "RDMA ERROR: RDMA hostname has not been set");
-}
+error_setg(errp, "RDMA ERROR: RDMA hostname has not been set");
 return -1;
 }
 
 /* create CM channel */
 rdma->channel = rdma_create_event_channel();
 if (!rdma->channel) {
-if (errp && !*errp) {
-error_setg(errp, "RDMA ERROR: could not create CM channel");
-}
+error_setg(errp, "RDMA ERROR: could not create CM channel");
 return -1;
 }
 
 /* create CM id */
 ret = rdma_create_id(rdma->channel, &rdma->cm_id, NULL, RDMA_PS_TCP);
 if (ret < 0) {
-if (errp && !*errp) {
-error_setg(errp, "RDMA ERROR: could not create channel id");
-}
+error_setg(errp, "RDMA ERROR: could not create channel id");
 goto err_resolve_create_id;
 }
 
@@ -970,10 +956,8 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error 
**errp)
 
 ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res);
 if (ret) {
-if (errp && !*errp) {
-error_setg(errp, "RDMA ERROR: could not rdma_getaddrinfo address 
%s",
-   rdma->host);
-}
+error_setg(errp, "RDMA ERROR: could not rdma_getaddrinfo address %s",
+   rdma->host);
 goto err_resolve_get_addr;
 }
 
@@ -1015,18 +999,14 @@ route:
 
 ret = rdma_get_cm_event(rdma->channel, &cm_event);
 if (ret < 0) {
-if (errp && !*errp) {
-error_setg(errp, "RDMA ERROR: could not perform 
event_addr_resolved");
-}
+error_setg(errp, "RDMA ERROR: could not perform event_addr_resolved");
 goto err_resolve_get_addr;
 }
 
 if (cm_event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
-if (errp && !*errp) {
-error_setg(errp,
- 

[PULL 31/65] migration/rdma: Fix rdma_getaddrinfo() error checking

[Juan Quintela]

From: Markus Armbruster 

rdma_getaddrinfo() returns 0 on success.  On error, it returns one of
the EAI_ error codes like getaddrinfo() does, or -1 with errno set.
This is broken by design: POSIX implicitly specifies the EAI_ error
codes to be non-zero, no more.  They could clash with -1.  Nothing we
can do about this design flaw.

Both callers of rdma_getaddrinfo() only recognize negative values as
error.  Works only because systems elect to make the EAI_ error codes
negative.

Best not to rely on that: change the callers to treat any non-zero
value as failure.  Also change them to return -1 instead of the value
received from getaddrinfo() on failure, to avoid positive error
values.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-25-arm...@redhat.com>
---
 migration/rdma.c | 14 ++
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 974edde6a3..dd0b073792 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -941,14 +941,14 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, 
Error **errp)
 
 if (rdma->host == NULL || !strcmp(rdma->host, "")) {
 ERROR(errp, "RDMA hostname has not been set");
-return -EINVAL;
+return -1;
 }
 
 /* create CM channel */
 rdma->channel = rdma_create_event_channel();
 if (!rdma->channel) {
 ERROR(errp, "could not create CM channel");
-return -EINVAL;
+return -1;
 }
 
 /* create CM id */
@@ -962,7 +962,7 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error 
**errp)
 port_str[15] = '\0';
 
 ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res);
-if (ret < 0) {
+if (ret) {
 ERROR(errp, "could not rdma_getaddrinfo address %s", rdma->host);
 goto err_resolve_get_addr;
 }
@@ -1004,7 +1004,6 @@ route:
 rdma_event_str(cm_event->event));
 error_report("rdma_resolve_addr");
 rdma_ack_cm_event(cm_event);
-ret = -EINVAL;
 goto err_resolve_get_addr;
 }
 rdma_ack_cm_event(cm_event);
@@ -1025,7 +1024,6 @@ route:
 ERROR(errp, "result not equal to event_route_resolved: %s",
 rdma_event_str(cm_event->event));
 rdma_ack_cm_event(cm_event);
-ret = -EINVAL;
 goto err_resolve_get_addr;
 }
 rdma_ack_cm_event(cm_event);
@@ -1040,7 +1038,7 @@ err_resolve_get_addr:
 err_resolve_create_id:
 rdma_destroy_event_channel(rdma->channel);
 rdma->channel = NULL;
-return ret;
+return -1;
 }
 
 /*
@@ -2695,7 +2693,7 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error 
**errp)
 port_str[15] = '\0';
 
 ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res);
-if (ret < 0) {
+if (ret) {
 ERROR(errp, "could not rdma_getaddrinfo address %s", rdma->host);
 goto err_dest_init_bind_addr;
 }
@@ -2739,7 +2737,7 @@ err_dest_init_create_listen_id:
 rdma_destroy_event_channel(rdma->channel);
 rdma->channel = NULL;
 rdma->error_state = ret;
-return ret;
+return -1;
 
 }
 
-- 
2.41.0

[PULL 28/65] migration/rdma: Fix qemu_get_cm_event_timeout() to always set error

[Juan Quintela]

From: Markus Armbruster 

qemu_get_cm_event_timeout() neglects to set an error when it fails
because rdma_get_cm_event() fails.  Harmless, as its caller
qemu_rdma_connect() substitutes a generic error then.  Fix it anyway.

qemu_rdma_connect() also sets the generic error when its own call of
rdma_get_cm_event() fails.  Make the error handling more obvious: set
a specific error right after rdma_get_cm_event() fails.  Delete the
generic error.

Signed-off-by: Markus Armbruster 
Reviewed-by: Peter Xu 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-22-arm...@redhat.com>
---
 migration/rdma.c | 10 --
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 9c576bdcba..0039295a15 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2555,7 +2555,11 @@ static int qemu_get_cm_event_timeout(RDMAContext *rdma,
 ERROR(errp, "failed to poll cm event, errno=%i", errno);
 return -1;
 } else if (poll_fd.revents & POLLIN) {
-return rdma_get_cm_event(rdma->channel, cm_event);
+if (rdma_get_cm_event(rdma->channel, cm_event) < 0) {
+ERROR(errp, "failed to get cm event");
+return -1;
+}
+return 0;
 } else {
 ERROR(errp, "no POLLIN event, revent=%x", poll_fd.revents);
 return -1;
@@ -2605,6 +2609,9 @@ static int qemu_rdma_connect(RDMAContext *rdma, bool 
return_path,
 ret = qemu_get_cm_event_timeout(rdma, &cm_event, 5000, errp);
 } else {
 ret = rdma_get_cm_event(rdma->channel, &cm_event);
+if (ret < 0) {
+ERROR(errp, "failed to get cm event");
+}
 }
 if (ret) {
 /*
@@ -2613,7 +2620,6 @@ static int qemu_rdma_connect(RDMAContext *rdma, bool 
return_path,
  * Will go away later in this series.
  */
 perror("rdma_get_cm_event after rdma_connect");
-ERROR(errp, "connecting to destination!");
 goto err_rdma_source_connect;
 }
 
-- 
2.41.0

[PULL 11/65] migration/rdma: Drop fragile wr_id formatting

[Juan Quintela]

From: Markus Armbruster 

wrid_desc[] uses 4001 pointers to map four integer values to strings.

print_wrid() accesses wrid_desc[] out of bounds when passed a negative
argument.  It returns null for values 2..1999 and 2001..3999.

qemu_rdma_poll() and qemu_rdma_block_for_wrid() print wrid_desc[wr_id]
and passes print_wrid(wr_id) to tracepoints.  Could conceivably crash
trying to format a null string.  I believe access out of bounds is not
possible.

Not worth cleaning up.  Dumb down to show just numeric wr_id.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-5-arm...@redhat.com>
---
 migration/rdma.c   | 32 +++-
 migration/trace-events |  8 
 2 files changed, 11 insertions(+), 29 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 65ed814d88..8f297c9e46 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -133,13 +133,6 @@ enum {
 RDMA_WRID_RECV_CONTROL = 4000,
 };
 
-static const char *wrid_desc[] = {
-[RDMA_WRID_NONE] = "NONE",
-[RDMA_WRID_RDMA_WRITE] = "WRITE RDMA",
-[RDMA_WRID_SEND_CONTROL] = "CONTROL SEND",
-[RDMA_WRID_RECV_CONTROL] = "CONTROL RECV",
-};
-
 /*
  * Work request IDs for IB SEND messages only (not RDMA writes).
  * This is used by the migration protocol to transmit
@@ -535,7 +528,6 @@ static void network_to_result(RDMARegisterResult *result)
 result->host_addr = ntohll(result->host_addr);
 };
 
-const char *print_wrid(int wrid);
 static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
uint8_t *data, RDMAControlHeader *resp,
int *resp_idx,
@@ -1362,14 +1354,6 @@ static int qemu_rdma_reg_control(RDMAContext *rdma, int 
idx)
 return -1;
 }
 
-const char *print_wrid(int wrid)
-{
-if (wrid >= RDMA_WRID_RECV_CONTROL) {
-return wrid_desc[RDMA_WRID_RECV_CONTROL];
-}
-return wrid_desc[wrid];
-}
-
 /*
  * Perform a non-optimized memory unregistration after every transfer
  * for demonstration purposes, only if pin-all is not requested.
@@ -1491,15 +1475,15 @@ static int qemu_rdma_poll(RDMAContext *rdma, struct 
ibv_cq *cq,
 if (wc.status != IBV_WC_SUCCESS) {
 fprintf(stderr, "ibv_poll_cq wc.status=%d %s!\n",
 wc.status, ibv_wc_status_str(wc.status));
-fprintf(stderr, "ibv_poll_cq wrid=%s!\n", wrid_desc[wr_id]);
+fprintf(stderr, "ibv_poll_cq wrid=%" PRIu64 "!\n", wr_id);
 
 return -1;
 }
 
 if (rdma->control_ready_expected &&
 (wr_id >= RDMA_WRID_RECV_CONTROL)) {
-trace_qemu_rdma_poll_recv(wrid_desc[RDMA_WRID_RECV_CONTROL],
-  wr_id - RDMA_WRID_RECV_CONTROL, wr_id, rdma->nb_sent);
+trace_qemu_rdma_poll_recv(wr_id - RDMA_WRID_RECV_CONTROL, wr_id,
+  rdma->nb_sent);
 rdma->control_ready_expected = 0;
 }
 
@@ -1510,7 +1494,7 @@ static int qemu_rdma_poll(RDMAContext *rdma, struct 
ibv_cq *cq,
 (wc.wr_id & RDMA_WRID_BLOCK_MASK) >> RDMA_WRID_BLOCK_SHIFT;
 RDMALocalBlock *block = &(rdma->local_ram_blocks.block[index]);
 
-trace_qemu_rdma_poll_write(print_wrid(wr_id), wr_id, rdma->nb_sent,
+trace_qemu_rdma_poll_write(wr_id, rdma->nb_sent,
index, chunk, block->local_host_addr,
(void *)(uintptr_t)block->remote_host_addr);
 
@@ -1520,7 +1504,7 @@ static int qemu_rdma_poll(RDMAContext *rdma, struct 
ibv_cq *cq,
 rdma->nb_sent--;
 }
 } else {
-trace_qemu_rdma_poll_other(print_wrid(wr_id), wr_id, rdma->nb_sent);
+trace_qemu_rdma_poll_other(wr_id, rdma->nb_sent);
 }
 
 *wr_id_out = wc.wr_id;
@@ -1665,8 +1649,7 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma, 
int wrid_requested,
 break;
 }
 if (wr_id != wrid_requested) {
-trace_qemu_rdma_block_for_wrid_miss(print_wrid(wrid_requested),
-   wrid_requested, print_wrid(wr_id), wr_id);
+trace_qemu_rdma_block_for_wrid_miss(wrid_requested, wr_id);
 }
 }
 
@@ -1705,8 +1688,7 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma, 
int wrid_requested,
 break;
 }
 if (wr_id != wrid_requested) {
-trace_qemu_rdma_block_for_wrid_miss(print_wrid(wrid_requested),
-   wrid_requested, print_wrid(wr_id), wr_id);
+trace_qemu_rdma_block_for_wrid_miss(wrid_requested, wr_id);
 }
 }
 
diff --git a/migration/trace-events b/migration/trace-events
index 002abe3a4e..19f9ee7c6d 100644
--- a/migration/trace-events
+++ b/migration/trace-events
@@ -208,7 +208,7 @@ qemu_rdma_accept_incoming_migration(void) ""
 qemu_rdma_accept_incoming_migration_ac

[PULL 56/65] migration/rdma: Silence qemu_rdma_block_for_wrid()

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_post_send_control(), qemu_rdma_exchange_get_response(), and
qemu_rdma_write_one() violate this principle: they call
error_report(), fprintf(stderr, ...), and perror() via
qemu_rdma_block_for_wrid(), qemu_rdma_poll(), and
qemu_rdma_wait_comp_channel().  I elected not to investigate how
callers handle the error, i.e. precise impact is not known.

Clean this up by dropping the error reporting from qemu_rdma_poll(),
qemu_rdma_wait_comp_channel(), and qemu_rdma_block_for_wrid().  I
believe the callers' error reports suffice.  If they don't, we need to
convert to Error instead.

Bonus: resolves a FIXME about problematic use of errno.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-50-arm...@redhat.com>
---
 migration/rdma.c | 16 
 1 file changed, 16 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index b5bb47108e..459dcb002e 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1483,17 +1483,12 @@ static int qemu_rdma_poll(RDMAContext *rdma, struct 
ibv_cq *cq,
 }
 
 if (ret < 0) {
-error_report("ibv_poll_cq failed");
 return -1;
 }
 
 wr_id = wc.wr_id & RDMA_WRID_TYPE_MASK;
 
 if (wc.status != IBV_WC_SUCCESS) {
-fprintf(stderr, "ibv_poll_cq wc.status=%d %s!\n",
-wc.status, ibv_wc_status_str(wc.status));
-fprintf(stderr, "ibv_poll_cq wrid=%" PRIu64 "!\n", wr_id);
-
 return -1;
 }
 
@@ -1577,16 +1572,12 @@ static int qemu_rdma_wait_comp_channel(RDMAContext 
*rdma,
 if (pfds[1].revents) {
 ret = rdma_get_cm_event(rdma->channel, &cm_event);
 if (ret < 0) {
-error_report("failed to get cm event while wait "
- "completion channel");
 return -1;
 }
 
 if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED ||
 cm_event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) {
 rdma_ack_cm_event(cm_event);
-error_report("receive cm event while wait comp 
channel,"
- "cm event is %d", cm_event->event);
 return -1;
 }
 rdma_ack_cm_event(cm_event);
@@ -1599,7 +1590,6 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma,
 default: /* Error of some type -
   * I don't trust errno from qemu_poll_ns
  */
-error_report("%s: poll failed", __func__);
 return -1;
 }
 
@@ -1683,12 +1673,6 @@ static int qemu_rdma_block_for_wrid(RDMAContext *rdma,
 
 ret = ibv_get_cq_event(ch, &cq, &cq_ctx);
 if (ret < 0) {
-/*
- * FIXME perror() is problematic, because ibv_reg_mr() is
- * not documented to set errno.  Will go away later in
- * this series.
- */
-perror("ibv_get_cq_event");
 goto err_block_for_wrid;
 }
 
-- 
2.41.0

[PULL 62/65] migration: Introduce migrate_has_error()

[Juan Quintela]

From: Peter Xu 

Introduce a helper to detect whether MigrationState.error is set for
whatever reason.

This is preparation work for any thread (e.g. source return path thread) to
setup errors in an unified way to MigrationState, rather than relying on
its own way to set errors (mark_source_rp_bad()).

Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Peter Xu 
Signed-off-by: Juan Quintela 
Message-ID: <20231004220240.167175-3-pet...@redhat.com>
---
 migration/migration.h | 1 +
 migration/migration.c | 7 +++
 2 files changed, 8 insertions(+)

diff --git a/migration/migration.h b/migration/migration.h
index 972597f4de..4106a1dc54 100644
--- a/migration/migration.h
+++ b/migration/migration.h
@@ -476,6 +476,7 @@ bool  migration_has_all_channels(void);
 uint64_t migrate_max_downtime(void);
 
 void migrate_set_error(MigrationState *s, const Error *error);
+bool migrate_has_error(MigrationState *s);
 
 void migrate_fd_connect(MigrationState *s, Error *error_in);
 
diff --git a/migration/migration.c b/migration/migration.c
index 57f9e9ed0c..409eb3e916 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -1234,6 +1234,13 @@ void migrate_set_error(MigrationState *s, const Error 
*error)
 }
 }
 
+bool migrate_has_error(MigrationState *s)
+{
+/* The lock is not helpful here, but still follow the rule */
+QEMU_LOCK_GUARD(&s->error_mutex);
+return qatomic_read(&s->error);
+}
+
 static void migrate_error_free(MigrationState *s)
 {
 QEMU_LOCK_GUARD(&s->error_mutex);
-- 
2.41.0

[PULL 61/65] migration: Display error in query-migrate irrelevant of status

[Juan Quintela]

From: Peter Xu 

Display it as long as being set, irrelevant of FAILED status.  E.g., it may
also be applicable to PAUSED stage of postcopy, to provide hint on what has
gone wrong.

The error_mutex seems to be overlooked when referencing the error, add it
to be very safe.

This will change QAPI behavior by showing up error message outside !FAILED
status, but it's intended and doesn't expect to break anyone.

Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2018404
Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Peter Xu 
Signed-off-by: Juan Quintela 
Message-ID: <20231004220240.167175-2-pet...@redhat.com>
---
 qapi/migration.json   | 5 ++---
 migration/migration.c | 8 +---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/qapi/migration.json b/qapi/migration.json
index d8f3bbd7b0..d7dfaa5db9 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -230,9 +230,8 @@
 # throttled during auto-converge.  This is only present when
 # auto-converge has started throttling guest cpus.  (Since 2.7)
 #
-# @error-desc: the human readable error description string, when
-# @status is 'failed'. Clients should not attempt to parse the
-# error strings.  (Since 2.7)
+# @error-desc: the human readable error description string. Clients
+# should not attempt to parse the error strings.  (Since 2.7)
 #
 # @postcopy-blocktime: total time when all vCPU were blocked during
 # postcopy live migration.  This is only present when the
diff --git a/migration/migration.c b/migration/migration.c
index 2057e42134..57f9e9ed0c 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -1060,9 +1060,6 @@ static void fill_source_migration_info(MigrationInfo 
*info)
 break;
 case MIGRATION_STATUS_FAILED:
 info->has_status = true;
-if (s->error) {
-info->error_desc = g_strdup(error_get_pretty(s->error));
-}
 break;
 case MIGRATION_STATUS_CANCELLED:
 info->has_status = true;
@@ -1072,6 +1069,11 @@ static void fill_source_migration_info(MigrationInfo 
*info)
 break;
 }
 info->status = state;
+
+QEMU_LOCK_GUARD(&s->error_mutex);
+if (s->error) {
+info->error_desc = g_strdup(error_get_pretty(s->error));
+}
 }
 
 static void fill_destination_migration_info(MigrationInfo *info)
-- 
2.41.0

[PULL 42/65] migration/rdma: Convert qemu_rdma_exchange_recv() to Error

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qio_channel_rdma_readv() violates this principle: it calls
error_report() via qemu_rdma_exchange_recv().  I elected not to
investigate how callers handle the error, i.e. precise impact is not
known.

Clean this up by converting qemu_rdma_exchange_recv() to Error.

Necessitates setting an error when qemu_rdma_exchange_get_response()
failed.  Since this error will go away later in this series, simply
use "FIXME temporary error message" there.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-36-arm...@redhat.com>
---
 migration/rdma.c | 15 +--
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index fdb527af39..b13fec6328 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1980,7 +1980,7 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
  * control-channel message.
  */
 static int qemu_rdma_exchange_recv(RDMAContext *rdma, RDMAControlHeader *head,
-   uint32_t expecting)
+   uint32_t expecting, Error **errp)
 {
 RDMAControlHeader ready = {
 .len = 0,
@@ -1995,7 +1995,7 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
 ret = qemu_rdma_post_send_control(rdma, NULL, &ready);
 
 if (ret < 0) {
-error_report("Failed to send control buffer!");
+error_setg(errp, "Failed to send control buffer!");
 return -1;
 }
 
@@ -2006,6 +2006,7 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
   expecting, RDMA_WRID_READY);
 
 if (ret < 0) {
+error_setg(errp, "FIXME temporary error message");
 return -1;
 }
 
@@ -2016,7 +2017,7 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
  */
 ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
 if (ret < 0) {
-error_report("rdma migration: error posting second control recv!");
+error_setg(errp, "rdma migration: error posting second control recv!");
 return -1;
 }
 
@@ -2957,11 +2958,11 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
 /* We've got nothing at all, so lets wait for
  * more to arrive
  */
-ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_QEMU_FILE);
+ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_QEMU_FILE,
+  errp);
 
 if (ret < 0) {
 rdma->errored = true;
-error_setg(errp, "qemu_rdma_exchange_recv failed");
 return -1;
 }
 
@@ -3575,6 +3576,7 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 RDMAControlHeader blocks = { .type = RDMA_CONTROL_RAM_BLOCKS_RESULT,
  .repeat = 1 };
 QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
+Error *err = NULL;
 RDMAContext *rdma;
 RDMALocalBlocks *local;
 RDMAControlHeader head;
@@ -3604,9 +3606,10 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 do {
 trace_qemu_rdma_registration_handle_wait();
 
-ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_NONE);
+ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_NONE, &err);
 
 if (ret < 0) {
+error_report_err(err);
 break;
 }
 
-- 
2.41.0

[PULL 40/65] migration/rdma: Fix error handling around rdma_getaddrinfo()

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_resolve_host() and qemu_rdma_dest_init() iterate over
addresses to find one that works, holding onto the first Error from
qemu_rdma_broken_ipv6_kernel() for use when no address works.  Issues:

1. If @errp was &error_abort or &error_fatal, we'd terminate instead
   of trying the next address.  Can't actually happen, since no caller
   passes these arguments.

2. When @errp is a pointer to a variable containing NULL, and
   qemu_rdma_broken_ipv6_kernel() fails, the variable no longer
   contains NULL.  Subsequent iterations pass it again, violating
   Error usage rules.  Dangerous, as setting an error would then trip
   error_setv()'s assertion.  Works only because
   qemu_rdma_broken_ipv6_kernel() and the code following the loops
   carefully avoids setting a second error.

3. If qemu_rdma_broken_ipv6_kernel() fails, and then a later iteration
   finds a working address, @errp still holds the first error from
   qemu_rdma_broken_ipv6_kernel().  If we then run into another error,
   we report the qemu_rdma_broken_ipv6_kernel() failure instead.

4. If we don't run into another error, we leak the Error object.

Use a local error variable, and propagate to @errp.  This fixes 3. and
also cleans up 1 and partly 2.

Free this error when we have a working address.  This fixes 4.

Pass the local error variable to qemu_rdma_broken_ipv6_kernel() only
until it fails.  Pass null on any later iterations.  This cleans up
the remainder of 2.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-34-arm...@redhat.com>
---
 migration/rdma.c | 25 +
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 4074509f06..3fb899f963 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -932,6 +932,7 @@ static int qemu_rdma_broken_ipv6_kernel(struct ibv_context 
*verbs, Error **errp)
  */
 static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
 {
+Error *err = NULL;
 int ret;
 struct rdma_addrinfo *res;
 char port_str[16];
@@ -976,7 +977,10 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error 
**errp)
 goto err_resolve_get_addr;
 }
 
+/* Try all addresses, saving the first error in @err */
 for (e = res; e != NULL; e = e->ai_next) {
+Error **local_errp = err ? NULL : &err;
+
 inet_ntop(e->ai_family,
 &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip);
 trace_qemu_rdma_resolve_host_trying(rdma->host, ip);
@@ -985,17 +989,21 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, 
Error **errp)
 RDMA_RESOLVE_TIMEOUT_MS);
 if (ret >= 0) {
 if (e->ai_family == AF_INET6) {
-ret = qemu_rdma_broken_ipv6_kernel(rdma->cm_id->verbs, errp);
+ret = qemu_rdma_broken_ipv6_kernel(rdma->cm_id->verbs,
+   local_errp);
 if (ret < 0) {
 continue;
 }
 }
+error_free(err);
 goto route;
 }
 }
 
 rdma_freeaddrinfo(res);
-if (errp && !*errp) {
+if (err) {
+error_propagate(errp, err);
+} else {
 error_setg(errp, "RDMA ERROR: could not resolve address %s",
rdma->host);
 }
@@ -2707,6 +2715,7 @@ err_rdma_source_connect:
 
 static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
 {
+Error *err = NULL;
 int ret, idx;
 struct rdma_cm_id *listen_id;
 char ip[40] = "unknown";
@@ -2765,7 +2774,11 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error 
**errp)
 }
 goto err_dest_init_bind_addr;
 }
+
+/* Try all addresses, saving the first error in @err */
 for (e = res; e != NULL; e = e->ai_next) {
+Error **local_errp = err ? NULL : &err;
+
 inet_ntop(e->ai_family,
 &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip);
 trace_qemu_rdma_dest_init_trying(rdma->host, ip);
@@ -2774,17 +2787,21 @@ static int qemu_rdma_dest_init(RDMAContext *rdma, Error 
**errp)
 continue;
 }
 if (e->ai_family == AF_INET6) {
-ret = qemu_rdma_broken_ipv6_kernel(listen_id->verbs, errp);
+ret = qemu_rdma_broken_ipv6_kernel(listen_id->verbs,
+   local_errp);
 if (ret < 0) {
 continue;
 }
 }
+error_free(err);
 break;
 }
 
 rdma_freeaddrinfo(res);
 if (!e) {
-if (errp && !*errp) {
+if (err) {
+error_propagate(errp, err);
+} else {
 error_setg(errp, "RDMA ERROR: Error: could not rdma_bind_addr!");
 }
 goto err_dest_init_bind_addr;
-- 
2.41.0

[PULL 10/65] migration/rdma: Clean up rdma_delete_block()'s return type

[Juan Quintela]

From: Markus Armbruster 

rdma_delete_block() always returns 0, which its only caller ignores.
Return void instead.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-4-arm...@redhat.com>
---
 migration/rdma.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 1432fb80ec..65ed814d88 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -668,7 +668,7 @@ static int qemu_rdma_init_ram_blocks(RDMAContext *rdma)
  * Note: If used outside of cleanup, the caller must ensure that the 
destination
  * block structures are also updated
  */
-static int rdma_delete_block(RDMAContext *rdma, RDMALocalBlock *block)
+static void rdma_delete_block(RDMAContext *rdma, RDMALocalBlock *block)
 {
 RDMALocalBlocks *local = &rdma->local_ram_blocks;
 RDMALocalBlock *old = local->block;
@@ -754,8 +754,6 @@ static int rdma_delete_block(RDMAContext *rdma, 
RDMALocalBlock *block)
 &local->block[x]);
 }
 }
-
-return 0;
 }
 
 /*
-- 
2.41.0

[PULL 21/65] migration/rdma: Make qemu_rdma_buffer_mergeable() return bool

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_buffer_mergeable() is semantically a predicate.  It returns
int 0 or 1.  Return bool instead, and fix the function name's
spelling.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-15-arm...@redhat.com>
---
 migration/rdma.c | 20 ++--
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index b412dad542..2e62d2cd0a 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2264,7 +2264,7 @@ static int qemu_rdma_write_flush(RDMAContext *rdma)
 return 0;
 }
 
-static inline int qemu_rdma_buffer_mergable(RDMAContext *rdma,
+static inline bool qemu_rdma_buffer_mergeable(RDMAContext *rdma,
 uint64_t offset, uint64_t len)
 {
 RDMALocalBlock *block;
@@ -2272,11 +2272,11 @@ static inline int qemu_rdma_buffer_mergable(RDMAContext 
*rdma,
 uint8_t *chunk_end;
 
 if (rdma->current_index < 0) {
-return 0;
+return false;
 }
 
 if (rdma->current_chunk < 0) {
-return 0;
+return false;
 }
 
 block = &(rdma->local_ram_blocks.block[rdma->current_index]);
@@ -2284,29 +2284,29 @@ static inline int qemu_rdma_buffer_mergable(RDMAContext 
*rdma,
 chunk_end = ram_chunk_end(block, rdma->current_chunk);
 
 if (rdma->current_length == 0) {
-return 0;
+return false;
 }
 
 /*
  * Only merge into chunk sequentially.
  */
 if (offset != (rdma->current_addr + rdma->current_length)) {
-return 0;
+return false;
 }
 
 if (offset < block->offset) {
-return 0;
+return false;
 }
 
 if ((offset + len) > (block->offset + block->length)) {
-return 0;
+return false;
 }
 
 if ((host_addr + len) > chunk_end) {
-return 0;
+return false;
 }
 
-return 1;
+return true;
 }
 
 /*
@@ -2329,7 +2329,7 @@ static int qemu_rdma_write(RDMAContext *rdma,
 int ret;
 
 /* If we cannot merge it, we flush the current buffer first. */
-if (!qemu_rdma_buffer_mergable(rdma, current_addr, len)) {
+if (!qemu_rdma_buffer_mergeable(rdma, current_addr, len)) {
 ret = qemu_rdma_write_flush(rdma);
 if (ret) {
 return ret;
-- 
2.41.0

[PULL 16/65] migration/rdma: Fix qemu_rdma_accept() to return failure on errors

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_accept() returns 0 in some cases even when it didn't
complete its job due to errors.  Impact is not obvious.  I figure the
caller will soon fail again with a misleading error message.

Fix it to return -1 on any failure.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-10-arm...@redhat.com>
---
 migration/rdma.c | 19 ---
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 1159f990af..8fd1b314b5 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -3360,6 +3360,7 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 
 if (cm_event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
 rdma_ack_cm_event(cm_event);
+ret = -1;
 goto err_rdma_dest_wait;
 }
 
@@ -3372,6 +3373,7 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 rdma_return_path = qemu_rdma_data_init(rdma->host_port, NULL);
 if (rdma_return_path == NULL) {
 rdma_ack_cm_event(cm_event);
+ret = -1;
 goto err_rdma_dest_wait;
 }
 
@@ -3383,10 +3385,11 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 network_to_caps(&cap);
 
 if (cap.version < 1 || cap.version > RDMA_CONTROL_VERSION_CURRENT) {
-error_report("Unknown source RDMA version: %d, bailing...",
-cap.version);
-rdma_ack_cm_event(cm_event);
-goto err_rdma_dest_wait;
+error_report("Unknown source RDMA version: %d, bailing...",
+ cap.version);
+rdma_ack_cm_event(cm_event);
+ret = -1;
+goto err_rdma_dest_wait;
 }
 
 /*
@@ -3416,9 +3419,10 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 if (!rdma->verbs) {
 rdma->verbs = verbs;
 } else if (rdma->verbs != verbs) {
-error_report("ibv context not matching %p, %p!", rdma->verbs,
- verbs);
-goto err_rdma_dest_wait;
+error_report("ibv context not matching %p, %p!", rdma->verbs,
+ verbs);
+ret = -1;
+goto err_rdma_dest_wait;
 }
 
 qemu_rdma_dump_id("dest_init", verbs);
@@ -3475,6 +3479,7 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) {
 error_report("rdma_accept not event established");
 rdma_ack_cm_event(cm_event);
+ret = -1;
 goto err_rdma_dest_wait;
 }
 
-- 
2.41.0

[PULL 39/65] migration/rdma: Retire macro ERROR()

[Juan Quintela]

From: Markus Armbruster 

ERROR() has become "error_setg() unless an error has been set
already".  Hiding the conditional in the macro is in the way of
further work.  Replace the macro uses by their expansion, and delete
the macro.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-33-arm...@redhat.com>
---
 migration/rdma.c | 168 +--
 1 file changed, 120 insertions(+), 48 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 6c0e6cda2c..4074509f06 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -40,13 +40,6 @@
 #include "options.h"
 #include 
 
-#define ERROR(errp, fmt, ...) \
-do { \
-if (errp && (*(errp) == NULL)) { \
-error_setg(errp, "RDMA ERROR: " fmt, ## __VA_ARGS__); \
-} \
-} while (0)
-
 #define RDMA_RESOLVE_TIMEOUT_MS 1
 
 /* Do not merge data if larger than this. */
@@ -865,7 +858,10 @@ static int qemu_rdma_broken_ipv6_kernel(struct ibv_context 
*verbs, Error **errp)
 
 if (ibv_query_port(verbs, 1, &port_attr)) {
 ibv_close_device(verbs);
-ERROR(errp, "Could not query initial IB port");
+if (errp && !*errp) {
+error_setg(errp,
+   "RDMA ERROR: Could not query initial IB port");
+}
 return -1;
 }
 
@@ -888,9 +884,12 @@ static int qemu_rdma_broken_ipv6_kernel(struct ibv_context 
*verbs, Error **errp)
 " migrate over the IB fabric until the kernel "
 " fixes the bug.\n");
 } else {
-ERROR(errp, "You only have RoCE / iWARP devices in your 
systems"
-" and your management software has specified 
'[::]'"
-", but IPv6 over RoCE / iWARP is not supported in 
Linux.");
+if (errp && !*errp) {
+error_setg(errp, "RDMA ERROR: "
+   "You only have RoCE / iWARP devices in your 
systems"
+   " and your management software has specified 
'[::]'"
+   ", but IPv6 over RoCE / iWARP is not supported 
in Linux.");
+}
 return -1;
 }
 }
@@ -906,13 +905,18 @@ static int qemu_rdma_broken_ipv6_kernel(struct 
ibv_context *verbs, Error **errp)
 
 /* IB ports start with 1, not 0 */
 if (ibv_query_port(verbs, 1, &port_attr)) {
-ERROR(errp, "Could not query initial IB port");
+if (errp && !*errp) {
+error_setg(errp, "RDMA ERROR: Could not query initial IB port");
+}
 return -1;
 }
 
 if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {
-ERROR(errp, "Linux kernel's RoCE / iWARP does not support IPv6 "
-"(but patches on linux-rdma in progress)");
+if (errp && !*errp) {
+error_setg(errp, "RDMA ERROR: "
+   "Linux kernel's RoCE / iWARP does not support IPv6 "
+   "(but patches on linux-rdma in progress)");
+}
 return -1;
 }
 
@@ -936,21 +940,27 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, 
Error **errp)
 struct rdma_addrinfo *e;
 
 if (rdma->host == NULL || !strcmp(rdma->host, "")) {
-ERROR(errp, "RDMA hostname has not been set");
+if (errp && !*errp) {
+error_setg(errp, "RDMA ERROR: RDMA hostname has not been set");
+}
 return -1;
 }
 
 /* create CM channel */
 rdma->channel = rdma_create_event_channel();
 if (!rdma->channel) {
-ERROR(errp, "could not create CM channel");
+if (errp && !*errp) {
+error_setg(errp, "RDMA ERROR: could not create CM channel");
+}
 return -1;
 }
 
 /* create CM id */
 ret = rdma_create_id(rdma->channel, &rdma->cm_id, NULL, RDMA_PS_TCP);
 if (ret < 0) {
-ERROR(errp, "could not create channel id");
+if (errp && !*errp) {
+error_setg(errp, "RDMA ERROR: could not create channel id");
+}
 goto err_resolve_create_id;
 }
 
@@ -959,7 +969,10 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error 
**errp)
 
 ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res);
 if (ret) {
-ERROR(errp, "could not rdma_getaddrinfo address %s", rdma->host);
+if (errp && !*errp) {
+error_setg(errp, "RDMA ERROR: could not rdma_getaddrinfo address 
%s",
+   rdma->host);
+}
 goto err_resolve_get_addr;
 }
 
@@ -982,7 +995,10 @@ static int qemu_rdma_resolve_host(RDMAContext *rdma, Error 
**errp)
 }
 
 rdma_freeaddrinfo(res);
-ERROR(errp, "could not resolve address %s", rdma->host);
+if (errp && !*errp) {
+er

[PULL 60/65] migration/rdma: Replace flawed device detail dump by tracing

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_dump_id() dumps RDMA device details to stdout.

rdma_start_outgoing_migration() calls it via qemu_rdma_source_init()
and qemu_rdma_resolve_host() to show source device details.
rdma_start_incoming_migration() arranges its call via
rdma_accept_incoming_migration() and qemu_rdma_accept() to show
destination device details.

Two issues:

1. rdma_start_outgoing_migration() can run in HMP context.  The
   information should arguably go the monitor, not stdout.

2. ibv_query_port() failure is reported as error.  Its callers remain
   unaware of this failure (qemu_rdma_dump_id() can't fail), so
   reporting this to the user as an error is problematic.

Fixable, but the device detail dump is noise, except when
troubleshooting.  Tracing is a better fit.  Similar function
qemu_rdma_dump_id() was converted to tracing in commit
733252deb8b (Tracify migration/rdma.c).

Convert qemu_rdma_dump_id(), too.

While there, touch up qemu_rdma_dump_gid()'s outdated comment.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-54-arm...@redhat.com>
---
 migration/rdma.c   | 23 ---
 migration/trace-events |  2 ++
 2 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index ab2ea85c45..f6fc226c9b 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -734,38 +734,31 @@ static void rdma_delete_block(RDMAContext *rdma, 
RDMALocalBlock *block)
 }
 
 /*
- * Put in the log file which RDMA device was opened and the details
- * associated with that device.
+ * Trace RDMA device open, with device details.
  */
 static void qemu_rdma_dump_id(const char *who, struct ibv_context *verbs)
 {
 struct ibv_port_attr port;
 
 if (ibv_query_port(verbs, 1, &port)) {
-error_report("Failed to query port information");
+trace_qemu_rdma_dump_id_failed(who);
 return;
 }
 
-printf("%s RDMA Device opened: kernel name %s "
-   "uverbs device name %s, "
-   "infiniband_verbs class device path %s, "
-   "infiniband class device path %s, "
-   "transport: (%d) %s\n",
-who,
+trace_qemu_rdma_dump_id(who,
 verbs->device->name,
 verbs->device->dev_name,
 verbs->device->dev_path,
 verbs->device->ibdev_path,
 port.link_layer,
-(port.link_layer == IBV_LINK_LAYER_INFINIBAND) ? "Infiniband" :
- ((port.link_layer == IBV_LINK_LAYER_ETHERNET)
-? "Ethernet" : "Unknown"));
+port.link_layer == IBV_LINK_LAYER_INFINIBAND ? "Infiniband"
+: port.link_layer == IBV_LINK_LAYER_ETHERNET ? "Ethernet"
+: "Unknown");
 }
 
 /*
- * Put in the log file the RDMA gid addressing information,
- * useful for folks who have trouble understanding the
- * RDMA device hierarchy in the kernel.
+ * Trace RDMA gid addressing information.
+ * Useful for understanding the RDMA device hierarchy in the kernel.
  */
 static void qemu_rdma_dump_gid(const char *who, struct rdma_cm_id *id)
 {
diff --git a/migration/trace-events b/migration/trace-events
index 6a50994402..ee9c8f4d63 100644
--- a/migration/trace-events
+++ b/migration/trace-events
@@ -214,6 +214,8 @@ qemu_rdma_close(void) ""
 qemu_rdma_connect_pin_all_requested(void) ""
 qemu_rdma_connect_pin_all_outcome(bool pin) "%d"
 qemu_rdma_dest_init_trying(const char *host, const char *ip) "%s => %s"
+qemu_rdma_dump_id_failed(const char *who) "%s RDMA Device opened, but can't 
query port information"
+qemu_rdma_dump_id(const char *who, const char *name, const char *dev_name, 
const char *dev_path, const char *ibdev_path, int transport, const char 
*transport_name) "%s RDMA Device opened: kernel name %s uverbs device name %s, 
infiniband_verbs class device path %s, infiniband class device path %s, 
transport: (%d) %s"
 qemu_rdma_dump_gid(const char *who, const char *src, const char *dst) "%s 
Source GID: %s, Dest GID: %s"
 qemu_rdma_exchange_get_response_start(const char *desc) "CONTROL: %s 
receiving..."
 qemu_rdma_exchange_get_response_none(const char *desc, int type) "Surprise: 
got %s (%d)"
-- 
2.41.0

[PULL 32/65] migration/rdma: Return -1 instead of negative errno code

[Juan Quintela]

From: Markus Armbruster 

Several functions return negative errno codes on failure.  Callers
check for specific codes exactly never.  For some of the functions,
callers couldn't check even if they wanted to, because the functions
also return negative values that aren't errno codes, leaving readers
confused on what the function actually returns.

Clean up and simplify: return -1 instead of negative errno code.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-26-arm...@redhat.com>
---
 migration/rdma.c | 44 ++--
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index dd0b073792..bc39b7ab2e 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -863,14 +863,14 @@ static int qemu_rdma_broken_ipv6_kernel(struct 
ibv_context *verbs, Error **errp)
 } else {
 error_setg_errno(errp, errno,
  "could not open RDMA device context");
-return -EINVAL;
+return -1;
 }
 }
 
 if (ibv_query_port(verbs, 1, &port_attr)) {
 ibv_close_device(verbs);
 ERROR(errp, "Could not query initial IB port");
-return -EINVAL;
+return -1;
 }
 
 if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) {
@@ -895,7 +895,7 @@ static int qemu_rdma_broken_ipv6_kernel(struct ibv_context 
*verbs, Error **errp)
 ERROR(errp, "You only have RoCE / iWARP devices in your 
systems"
 " and your management software has specified 
'[::]'"
 ", but IPv6 over RoCE / iWARP is not supported in 
Linux.");
-return -ENONET;
+return -1;
 }
 }
 
@@ -911,13 +911,13 @@ static int qemu_rdma_broken_ipv6_kernel(struct 
ibv_context *verbs, Error **errp)
 /* IB ports start with 1, not 0 */
 if (ibv_query_port(verbs, 1, &port_attr)) {
 ERROR(errp, "Could not query initial IB port");
-return -EINVAL;
+return -1;
 }
 
 if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {
 ERROR(errp, "Linux kernel's RoCE / iWARP does not support IPv6 "
 "(but patches on linux-rdma in progress)");
-return -ENONET;
+return -1;
 }
 
 #endif
@@ -1425,7 +1425,7 @@ static int qemu_rdma_unregister_waiting(RDMAContext *rdma)
  * this series.
  */
 perror("unregistration chunk failed");
-return -ret;
+return -1;
 }
 rdma->total_registrations--;
 
@@ -1570,7 +1570,7 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma,
 if (ret) {
 error_report("failed to get cm event while wait "
  "completion channel");
-return -EPIPE;
+return -1;
 }
 
 error_report("receive cm event while wait comp channel,"
@@ -1578,7 +1578,7 @@ static int qemu_rdma_wait_comp_channel(RDMAContext *rdma,
 if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED ||
 cm_event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) {
 rdma_ack_cm_event(cm_event);
-return -EPIPE;
+return -1;
 }
 rdma_ack_cm_event(cm_event);
 }
@@ -1591,18 +1591,18 @@ static int qemu_rdma_wait_comp_channel(RDMAContext 
*rdma,
   * I don't trust errno from qemu_poll_ns
  */
 error_report("%s: poll failed", __func__);
-return -EPIPE;
+return -1;
 }
 
 if (migrate_get_current()->state == MIGRATION_STATUS_CANCELLING) {
 /* Bail out and let the cancellation happen */
-return -EPIPE;
+return -1;
 }
 }
 }
 
 if (rdma->received_error) {
-return -EPIPE;
+return -1;
 }
 return rdma->error_state;
 }
@@ -1772,7 +1772,7 @@ static int qemu_rdma_post_send_control(RDMAContext *rdma, 
uint8_t *buf,
 
 if (ret > 0) {
 error_report("Failed to use post IB SEND for control");
-return -ret;
+return -1;
 }
 
 ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_SEND_CONTROL, NULL);
@@ -1841,15 +1841,15 @@ static int qemu_rdma_exchange_get_response(RDMAContext 
*rdma,
 if (head->type == RDMA_CONTROL_ERROR) {
 rdma->received_error = true;
 }
-return -EIO;
+return -1;
 }
 if (head->len > RDMA_CONTROL_MAX_BUFFER - sizeof(*head)) {
 error_report("too long leng

[PULL 63/65] qemufile: Always return a verbose error

[Juan Quintela]

From: Peter Xu 

There're a lot of cases where we only have an errno set in last_error but
without a detailed error description.  When this happens, try to generate
an error contains the errno as a descriptive error.

This will be helpful in cases where one relies on the Error*.  E.g.,
migration state only caches Error* in MigrationState.error.  With this,
we'll display correct error messages in e.g. query-migrate when the error
was only set by qemu_file_set_error().

Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Peter Xu 
Signed-off-by: Juan Quintela 
Message-ID: <20231004220240.167175-6-pet...@redhat.com>
---
 migration/qemu-file.c | 15 ---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/migration/qemu-file.c b/migration/qemu-file.c
index 5e8207dae4..7fb659296f 100644
--- a/migration/qemu-file.c
+++ b/migration/qemu-file.c
@@ -142,15 +142,24 @@ void qemu_file_set_hooks(QEMUFile *f, const QEMUFileHooks 
*hooks)
  *
  * Return negative error value if there has been an error on previous
  * operations, return 0 if no error happened.
- * Optional, it returns Error* in errp, but it may be NULL even if return value
- * is not 0.
  *
+ * If errp is specified, a verbose error message will be copied over.
  */
 static int qemu_file_get_error_obj(QEMUFile *f, Error **errp)
 {
+if (!f->last_error) {
+return 0;
+}
+
+/* There is an error */
 if (errp) {
-*errp = f->last_error_obj ? error_copy(f->last_error_obj) : NULL;
+if (f->last_error_obj) {
+*errp = error_copy(f->last_error_obj);
+} else {
+error_setg_errno(errp, -f->last_error, "Channel error");
+}
 }
+
 return f->last_error;
 }
 
-- 
2.41.0

[PULL 45/65] migration/rdma: Convert qemu_rdma_reg_whole_ram_blocks() to Error

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_exchange_send() violates this principle: it calls
error_report() via callback qemu_rdma_reg_whole_ram_blocks().  I
elected not to investigate how callers handle the error, i.e. precise
impact is not known.

Clean this up by converting the callback to Error.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-39-arm...@redhat.com>
---
 migration/rdma.c | 26 +-
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 87f2e6129e..189932097e 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -518,7 +518,8 @@ static void network_to_result(RDMARegisterResult *result)
 static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
uint8_t *data, RDMAControlHeader *resp,
int *resp_idx,
-   int (*callback)(RDMAContext *rdma),
+   int (*callback)(RDMAContext *rdma,
+   Error **errp),
Error **errp);
 
 static inline uint64_t ram_chunk_index(const uint8_t *start,
@@ -1177,7 +1178,7 @@ static void qemu_rdma_advise_prefetch_mr(struct ibv_pd 
*pd, uint64_t addr,
 #endif
 }
 
-static int qemu_rdma_reg_whole_ram_blocks(RDMAContext *rdma)
+static int qemu_rdma_reg_whole_ram_blocks(RDMAContext *rdma, Error **errp)
 {
 int i;
 RDMALocalBlocks *local = &rdma->local_ram_blocks;
@@ -1217,16 +1218,16 @@ static int qemu_rdma_reg_whole_ram_blocks(RDMAContext 
*rdma)
 }
 
 if (!local->block[i].mr) {
-perror("Failed to register local dest ram block!");
-break;
+error_setg_errno(errp, errno,
+ "Failed to register local dest ram block!");
+goto err;
 }
 rdma->total_registrations++;
 }
 
-if (i >= local->nb_blocks) {
-return 0;
-}
+return 0;
 
+err:
 for (i--; i >= 0; i--) {
 ibv_dereg_mr(local->block[i].mr);
 local->block[i].mr = NULL;
@@ -1899,7 +1900,8 @@ static void qemu_rdma_move_header(RDMAContext *rdma, int 
idx,
 static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
uint8_t *data, RDMAControlHeader *resp,
int *resp_idx,
-   int (*callback)(RDMAContext *rdma),
+   int (*callback)(RDMAContext *rdma,
+   Error **errp),
Error **errp)
 {
 int ret;
@@ -1956,9 +1958,8 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 if (resp) {
 if (callback) {
 trace_qemu_rdma_exchange_send_issue_callback();
-ret = callback(rdma);
+ret = callback(rdma, errp);
 if (ret < 0) {
-error_setg(errp, "FIXME temporary error message");
 return -1;
 }
 }
@@ -3671,10 +3672,9 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 }
 
 if (rdma->pin_all) {
-ret = qemu_rdma_reg_whole_ram_blocks(rdma);
+ret = qemu_rdma_reg_whole_ram_blocks(rdma, &err);
 if (ret < 0) {
-error_report("rdma migration: error dest "
-"registering ram blocks");
+error_report_err(err);
 goto err;
 }
 }
-- 
2.41.0

[PULL 44/65] migration/rdma: Convert qemu_rdma_exchange_get_response() to Error

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_exchange_send() and qemu_rdma_exchange_recv() violate this
principle: they call error_report() via
qemu_rdma_exchange_get_response().  I elected not to investigate how
callers handle the error, i.e. precise impact is not known.

Clean this up by converting qemu_rdma_exchange_get_response() to
Error.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-38-arm...@redhat.com>
---
 migration/rdma.c | 22 +++---
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 7866cf9bb7..87f2e6129e 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1824,14 +1824,15 @@ static int qemu_rdma_post_recv_control(RDMAContext 
*rdma, int idx)
  * Block and wait for a RECV control channel message to arrive.
  */
 static int qemu_rdma_exchange_get_response(RDMAContext *rdma,
-RDMAControlHeader *head, uint32_t expecting, int idx)
+RDMAControlHeader *head, uint32_t expecting, int idx,
+Error **errp)
 {
 uint32_t byte_len;
 int ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RECV_CONTROL + idx,
&byte_len);
 
 if (ret < 0) {
-error_report("rdma migration: recv polling control error!");
+error_setg(errp, "rdma migration: recv polling control error!");
 return -1;
 }
 
@@ -1844,7 +1845,7 @@ static int qemu_rdma_exchange_get_response(RDMAContext 
*rdma,
 trace_qemu_rdma_exchange_get_response_none(control_desc(head->type),
  head->type);
 } else if (head->type != expecting || head->type == RDMA_CONTROL_ERROR) {
-error_report("Was expecting a %s (%d) control message"
+error_setg(errp, "Was expecting a %s (%d) control message"
 ", but got: %s (%d), length: %d",
 control_desc(expecting), expecting,
 control_desc(head->type), head->type, head->len);
@@ -1854,11 +1855,12 @@ static int qemu_rdma_exchange_get_response(RDMAContext 
*rdma,
 return -1;
 }
 if (head->len > RDMA_CONTROL_MAX_BUFFER - sizeof(*head)) {
-error_report("too long length: %d", head->len);
+error_setg(errp, "too long length: %d", head->len);
 return -1;
 }
 if (sizeof(*head) + head->len != byte_len) {
-error_report("Malformed length: %d byte_len %d", head->len, byte_len);
+error_setg(errp, "Malformed length: %d byte_len %d",
+   head->len, byte_len);
 return -1;
 }
 
@@ -1911,9 +1913,8 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 
 ret = qemu_rdma_exchange_get_response(rdma, &resp_ignored,
   RDMA_CONTROL_READY,
-  RDMA_WRID_READY);
+  RDMA_WRID_READY, errp);
 if (ret < 0) {
-error_setg(errp, "FIXME temporary error message");
 return -1;
 }
 }
@@ -1964,10 +1965,10 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 
 trace_qemu_rdma_exchange_send_waiting(control_desc(resp->type));
 ret = qemu_rdma_exchange_get_response(rdma, resp,
-  resp->type, RDMA_WRID_DATA);
+  resp->type, RDMA_WRID_DATA,
+  errp);
 
 if (ret < 0) {
-error_setg(errp, "FIXME temporary error message");
 return -1;
 }
 
@@ -2011,10 +2012,9 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
  * Block and wait for the message.
  */
 ret = qemu_rdma_exchange_get_response(rdma, head,
-  expecting, RDMA_WRID_READY);
+  expecting, RDMA_WRID_READY, errp);
 
 if (ret < 0) {
-error_setg(errp, "FIXME temporary error message");
 return -1;
 }
 
-- 
2.41.0

[PULL 27/65] migration/rdma: Fix qemu_rdma_broken_ipv6_kernel() to set error

[Juan Quintela]

From: Markus Armbruster 

qemu_rdma_resolve_host() and qemu_rdma_dest_init() try addresses until
they find on that works.  If none works, they return the first Error
set by qemu_rdma_broken_ipv6_kernel(), or else return a generic one.

qemu_rdma_broken_ipv6_kernel() neglects to set an Error when
ibv_open_device() fails.  If a later address fails differently, we use
that Error instead, or else the generic one.  Harmless enough, but
needs fixing all the same.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-21-arm...@redhat.com>
---
 migration/rdma.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/migration/rdma.c b/migration/rdma.c
index 30e2c817f2..9c576bdcba 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -861,6 +861,8 @@ static int qemu_rdma_broken_ipv6_kernel(struct ibv_context 
*verbs, Error **errp)
 if (errno == EPERM) {
 continue;
 } else {
+error_setg_errno(errp, errno,
+ "could not open RDMA device context");
 return -EINVAL;
 }
 }
-- 
2.41.0

[PULL 59/65] migration/rdma: Use error_report() & friends instead of stderr

[Juan Quintela]

From: Markus Armbruster 

error_report() obeys -msg, reports the current error location if any,
and reports to the current monitor if any.  Reporting to stderr
directly with fprintf() or perror() is wrong, because it loses all
this.

Fix the offenders.  Bonus: resolves a FIXME about problematic use of
errno.

Signed-off-by: Markus Armbruster 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-53-arm...@redhat.com>
---
 migration/rdma.c | 44 +---
 1 file changed, 21 insertions(+), 23 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 9d5b3b76eb..ab2ea85c45 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -877,12 +877,12 @@ static int qemu_rdma_broken_ipv6_kernel(struct 
ibv_context *verbs, Error **errp)
 
 if (roce_found) {
 if (ib_found) {
-fprintf(stderr, "WARN: migrations may fail:"
-" IPv6 over RoCE / iWARP in linux"
-" is broken. But since you appear to have a"
-" mixed RoCE / IB environment, be sure to only"
-" migrate over the IB fabric until the kernel "
-" fixes the bug.\n");
+warn_report("migrations may fail:"
+" IPv6 over RoCE / iWARP in linux"
+" is broken. But since you appear to have a"
+" mixed RoCE / IB environment, be sure to only"
+" migrate over the IB fabric until the kernel "
+" fixes the bug.");
 } else {
 error_setg(errp, "RDMA ERROR: "
"You only have RoCE / iWARP devices in your systems"
@@ -1418,12 +1418,8 @@ static int qemu_rdma_unregister_waiting(RDMAContext 
*rdma)
 block->remote_keys[chunk] = 0;
 
 if (ret != 0) {
-/*
- * FIXME perror() is problematic, bcause ibv_dereg_mr() is
- * not documented to set errno.  Will go away later in
- * this series.
- */
-perror("unregistration chunk failed");
+error_report("unregistration chunk failed: %s",
+ strerror(ret));
 return -1;
 }
 rdma->total_registrations--;
@@ -3772,7 +3768,8 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 block->pmr[reg->key.chunk] = NULL;
 
 if (ret != 0) {
-perror("rdma unregistration chunk failed");
+error_report("rdma unregistration chunk failed: %s",
+ strerror(errno));
 goto err;
 }
 
@@ -3961,10 +3958,10 @@ static int qemu_rdma_registration_stop(QEMUFile *f,
  */
 
 if (local->nb_blocks != nb_dest_blocks) {
-fprintf(stderr, "ram blocks mismatch (Number of blocks %d vs %d) "
-"Your QEMU command line parameters are probably "
-"not identical on both the source and destination.",
-local->nb_blocks, nb_dest_blocks);
+error_report("ram blocks mismatch (Number of blocks %d vs %d)",
+ local->nb_blocks, nb_dest_blocks);
+error_printf("Your QEMU command line parameters are probably "
+ "not identical on both the source and destination.");
 rdma->errored = true;
 return -1;
 }
@@ -3977,10 +3974,11 @@ static int qemu_rdma_registration_stop(QEMUFile *f,
 
 /* We require that the blocks are in the same order */
 if (rdma->dest_blocks[i].length != local->block[i].length) {
-fprintf(stderr, "Block %s/%d has a different length %" PRIu64
-"vs %" PRIu64, local->block[i].block_name, i,
-local->block[i].length,
-rdma->dest_blocks[i].length);
+error_report("Block %s/%d has a different length %" PRIu64
+ "vs %" PRIu64,
+ local->block[i].block_name, i,
+ local->block[i].length,
+ rdma->dest_blocks[i].length);
 rdma->errored = true;
 return -1;
 }
@@ -4096,7 +4094,7 @@ static void rdma_accept_incoming_migration(void *opaque)
 ret = qemu_rdma_accept(rdma);
 
 if (ret < 0) {
-fprintf(stderr, "RDMA ERROR: Migration initialization failed\n");
+error_report("RDMA ERROR: Migration initialization failed");
 return;
 }
 
@@ -4108,7 +4106,7 @@ static void rdma_accept_incoming_migration(void *opaque)
 
 f = rdma_new_input(rdma);
 if (f == NULL) {
-fprintf(stderr, "RDMA ERROR: could not open RDMA for input\n");
+err

[PULL 49/65] migration/rdma: Convert qemu_rdma_post_send_control() to Error

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_exchange_send() violates this principle: it calls
error_report() via qemu_rdma_post_send_control().  I elected not to
investigate how callers handle the error, i.e. precise impact is not
known.

Clean this up by converting qemu_rdma_post_send_control() to Error.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-43-arm...@redhat.com>
---
 migration/rdma.c | 31 +--
 1 file changed, 17 insertions(+), 14 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 9f45f6a91d..aeb0a8921e 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1741,7 +1741,8 @@ err_block_for_wrid:
  * containing some data and block until the post completes.
  */
 static int qemu_rdma_post_send_control(RDMAContext *rdma, uint8_t *buf,
-   RDMAControlHeader *head)
+   RDMAControlHeader *head,
+   Error **errp)
 {
 int ret;
 RDMAWorkRequestData *wr = &rdma->wr_data[RDMA_WRID_CONTROL];
@@ -1781,13 +1782,13 @@ static int qemu_rdma_post_send_control(RDMAContext 
*rdma, uint8_t *buf,
 ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr);
 
 if (ret > 0) {
-error_report("Failed to use post IB SEND for control");
+error_setg(errp, "Failed to use post IB SEND for control");
 return -1;
 }
 
 ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_SEND_CONTROL, NULL);
 if (ret < 0) {
-error_report("rdma migration: send polling control error");
+error_setg(errp, "rdma migration: send polling control error");
 return -1;
 }
 
@@ -1945,10 +1946,9 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 /*
  * Deliver the control message that was requested.
  */
-ret = qemu_rdma_post_send_control(rdma, data, head);
+ret = qemu_rdma_post_send_control(rdma, data, head, errp);
 
 if (ret < 0) {
-error_setg(errp, "Failed to send control buffer!");
 return -1;
 }
 
@@ -2002,10 +2002,9 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
 /*
  * Inform the source that we're ready to receive a message.
  */
-ret = qemu_rdma_post_send_control(rdma, NULL, &ready);
+ret = qemu_rdma_post_send_control(rdma, NULL, &ready, errp);
 
 if (ret < 0) {
-error_setg(errp, "Failed to send control buffer!");
 return -1;
 }
 
@@ -2394,6 +2393,7 @@ static int qemu_rdma_write(RDMAContext *rdma,
 
 static void qemu_rdma_cleanup(RDMAContext *rdma)
 {
+Error *err = NULL;
 int idx;
 
 if (rdma->cm_id && rdma->connected) {
@@ -2405,7 +2405,9 @@ static void qemu_rdma_cleanup(RDMAContext *rdma)
.repeat = 1,
  };
 error_report("Early error. Sending error.");
-qemu_rdma_post_send_control(rdma, NULL, &head);
+if (qemu_rdma_post_send_control(rdma, NULL, &head, &err) < 0) {
+error_report_err(err);
+}
 }
 
 rdma_disconnect(rdma->cm_id);
@@ -3705,10 +3707,11 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 
 
 ret = qemu_rdma_post_send_control(rdma,
-(uint8_t *) rdma->dest_blocks, 
&blocks);
+(uint8_t *) rdma->dest_blocks, &blocks,
+&err);
 
 if (ret < 0) {
-error_report("rdma migration: error sending remote info");
+error_report_err(err);
 goto err;
 }
 
@@ -3783,10 +3786,10 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
 }
 
 ret = qemu_rdma_post_send_control(rdma,
-(uint8_t *) results, ®_resp);
+(uint8_t *) results, ®_resp, &err);
 
 if (ret < 0) {
-error_report("Failed to send control buffer");
+error_report_err(err);
 goto err;
 }
 break;
@@ -3818,10 +3821,10 @@ static int qemu_rdma_registration_handle(QEMUFile *f)
reg->key.chunk);
 }
 
-ret = qemu_rdma_post_send_control(rdma, NULL, &unreg_resp);
+ret = qemu_rdma_post_send_control(rdma, NULL, &unreg_resp, &err);
 
 if (ret < 0) {
-error_report("Failed to send control buffer");
+

[PULL 18/65] migration/rdma: Eliminate error_propagate()

[Juan Quintela]

From: Markus Armbruster 

When all we do with an Error we receive into a local variable is
propagating to somewhere else, we can just as well receive it there
right away.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-12-arm...@redhat.com>
---
 migration/rdma.c | 19 +++
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index bc6d8248f2..c858d3fbe4 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2465,7 +2465,6 @@ static void qemu_rdma_cleanup(RDMAContext *rdma)
 static int qemu_rdma_source_init(RDMAContext *rdma, bool pin_all, Error **errp)
 {
 int ret, idx;
-Error *local_err = NULL, **temp = &local_err;
 
 /*
  * Will be validated against destination's actual capabilities
@@ -2473,14 +2472,14 @@ static int qemu_rdma_source_init(RDMAContext *rdma, 
bool pin_all, Error **errp)
  */
 rdma->pin_all = pin_all;
 
-ret = qemu_rdma_resolve_host(rdma, temp);
+ret = qemu_rdma_resolve_host(rdma, errp);
 if (ret) {
 goto err_rdma_source_init;
 }
 
 ret = qemu_rdma_alloc_pd_cq(rdma);
 if (ret) {
-ERROR(temp, "rdma migration: error allocating pd and cq! Your mlock()"
+ERROR(errp, "rdma migration: error allocating pd and cq! Your mlock()"
 " limits may be too low. Please check $ ulimit -a # and "
 "search for 'ulimit -l' in the output");
 goto err_rdma_source_init;
@@ -2488,13 +2487,13 @@ static int qemu_rdma_source_init(RDMAContext *rdma, 
bool pin_all, Error **errp)
 
 ret = qemu_rdma_alloc_qp(rdma);
 if (ret) {
-ERROR(temp, "rdma migration: error allocating qp!");
+ERROR(errp, "rdma migration: error allocating qp!");
 goto err_rdma_source_init;
 }
 
 ret = qemu_rdma_init_ram_blocks(rdma);
 if (ret) {
-ERROR(temp, "rdma migration: error initializing ram blocks!");
+ERROR(errp, "rdma migration: error initializing ram blocks!");
 goto err_rdma_source_init;
 }
 
@@ -2509,7 +2508,7 @@ static int qemu_rdma_source_init(RDMAContext *rdma, bool 
pin_all, Error **errp)
 for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
 ret = qemu_rdma_reg_control(rdma, idx);
 if (ret) {
-ERROR(temp, "rdma migration: error registering %d control!",
+ERROR(errp, "rdma migration: error registering %d control!",
 idx);
 goto err_rdma_source_init;
 }
@@ -2518,7 +2517,6 @@ static int qemu_rdma_source_init(RDMAContext *rdma, bool 
pin_all, Error **errp)
 return 0;
 
 err_rdma_source_init:
-error_propagate(errp, local_err);
 qemu_rdma_cleanup(rdma);
 return -1;
 }
@@ -4111,7 +4109,6 @@ void rdma_start_incoming_migration(const char *host_port, 
Error **errp)
 {
 int ret;
 RDMAContext *rdma;
-Error *local_err = NULL;
 
 trace_rdma_start_incoming_migration();
 
@@ -4121,13 +4118,12 @@ void rdma_start_incoming_migration(const char 
*host_port, Error **errp)
 return;
 }
 
-rdma = qemu_rdma_data_init(host_port, &local_err);
+rdma = qemu_rdma_data_init(host_port, errp);
 if (rdma == NULL) {
 goto err;
 }
 
-ret = qemu_rdma_dest_init(rdma, &local_err);
-
+ret = qemu_rdma_dest_init(rdma, errp);
 if (ret) {
 goto err;
 }
@@ -4150,7 +4146,6 @@ void rdma_start_incoming_migration(const char *host_port, 
Error **errp)
 cleanup_rdma:
 qemu_rdma_cleanup(rdma);
 err:
-error_propagate(errp, local_err);
 if (rdma) {
 g_free(rdma->host);
 g_free(rdma->host_port);
-- 
2.41.0

[PULL 02/65] tests/qtest: migration: Expose migrate_set_capability

[Juan Quintela]

From: Fabiano Rosas 

The following patch will make use of this function from within
migrate-helpers.c, so move it there.

Reviewed-by: Juan Quintela 
Reviewed-by: Thomas Huth 
Reviewed-by: Peter Xu 
Signed-off-by: Fabiano Rosas 
Signed-off-by: Juan Quintela 
Message-ID: <20230712190742.22294-2-faro...@suse.de>
---
 tests/qtest/migration-helpers.h |  3 +++
 tests/qtest/migration-helpers.c | 11 +++
 tests/qtest/migration-test.c| 11 ---
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/tests/qtest/migration-helpers.h b/tests/qtest/migration-helpers.h
index 009e250e90..484d7c960f 100644
--- a/tests/qtest/migration-helpers.h
+++ b/tests/qtest/migration-helpers.h
@@ -23,6 +23,9 @@ bool migrate_watch_for_resume(QTestState *who, const char 
*name,
 G_GNUC_PRINTF(3, 4)
 void migrate_qmp(QTestState *who, const char *uri, const char *fmt, ...);
 
+void migrate_set_capability(QTestState *who, const char *capability,
+bool value);
+
 QDict *migrate_query(QTestState *who);
 QDict *migrate_query_not_failed(QTestState *who);
 
diff --git a/tests/qtest/migration-helpers.c b/tests/qtest/migration-helpers.c
index be00c52d00..2df198c99e 100644
--- a/tests/qtest/migration-helpers.c
+++ b/tests/qtest/migration-helpers.c
@@ -70,6 +70,17 @@ void migrate_qmp(QTestState *who, const char *uri, const 
char *fmt, ...)
  "{ 'execute': 'migrate', 'arguments': %p}", args);
 }
 
+void migrate_set_capability(QTestState *who, const char *capability,
+bool value)
+{
+qtest_qmp_assert_success(who,
+ "{ 'execute': 'migrate-set-capabilities',"
+ "'arguments': { "
+ "'capabilities': [ { "
+ "'capability': %s, 'state': %i } ] } }",
+ capability, value);
+}
+
 /*
  * Note: caller is responsible to free the returned object via
  * qobject_unref() after use
diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c
index 46f1c275a2..48778565a3 100644
--- a/tests/qtest/migration-test.c
+++ b/tests/qtest/migration-test.c
@@ -605,17 +605,6 @@ static void migrate_cancel(QTestState *who)
 qtest_qmp_assert_success(who, "{ 'execute': 'migrate_cancel' }");
 }
 
-static void migrate_set_capability(QTestState *who, const char *capability,
-   bool value)
-{
-qtest_qmp_assert_success(who,
- "{ 'execute': 'migrate-set-capabilities',"
- "'arguments': { "
- "'capabilities': [ { "
- "'capability': %s, 'state': %i } ] } }",
- capability, value);
-}
-
 static void migrate_postcopy_start(QTestState *from, QTestState *to)
 {
 qtest_qmp_assert_success(from, "{ 'execute': 'migrate-start-postcopy' }");
-- 
2.41.0

[PULL 58/65] migration/rdma: Downgrade qemu_rdma_cleanup() errors to warnings

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_source_init(), qemu_rdma_connect(),
rdma_start_incoming_migration(), and rdma_start_outgoing_migration()
violate this principle: they call error_report() via
qemu_rdma_cleanup().

Moreover, qemu_rdma_cleanup() can't fail.  It is called on error
paths, and QIOChannel close and finalization.  Are the conditions it
reports really errors?  I doubt it.

Downgrade qemu_rdma_cleanup()'s errors to warnings.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-52-arm...@redhat.com>
---
 migration/rdma.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 025523bf70..9d5b3b76eb 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2375,9 +2375,9 @@ static void qemu_rdma_cleanup(RDMAContext *rdma)
.type = RDMA_CONTROL_ERROR,
.repeat = 1,
  };
-error_report("Early error. Sending error.");
+warn_report("Early error. Sending error.");
 if (qemu_rdma_post_send_control(rdma, NULL, &head, &err) < 0) {
-error_report_err(err);
+warn_report_err(err);
 }
 }
 
-- 
2.41.0

[PULL 53/65] migration/rdma: Silence qemu_rdma_connect()

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_connect() violates this principle: it calls error_report()
and perror().  I elected not to investigate how callers handle the
error, i.e. precise impact is not known.

Clean this up: replace perror() by changing error_setg() to
error_setg_errno(), and drop error_report().  I believe the callers'
error reports suffice then.  If they don't, we need to convert to
Error instead.

Bonus: resolves a FIXME about problematic use of errno.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-47-arm...@redhat.com>
---
 migration/rdma.c | 14 --
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 3c7a407d25..8e1e8c4d47 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2608,8 +2608,8 @@ static int qemu_rdma_connect(RDMAContext *rdma, bool 
return_path,
 
 ret = rdma_connect(rdma->cm_id, &conn_param);
 if (ret < 0) {
-perror("rdma_connect");
-error_setg(errp, "RDMA ERROR: connecting to destination!");
+error_setg_errno(errp, errno,
+ "RDMA ERROR: connecting to destination!");
 goto err_rdma_source_connect;
 }
 
@@ -2618,21 +2618,15 @@ static int qemu_rdma_connect(RDMAContext *rdma, bool 
return_path,
 } else {
 ret = rdma_get_cm_event(rdma->channel, &cm_event);
 if (ret < 0) {
-error_setg(errp, "RDMA ERROR: failed to get cm event");
+error_setg_errno(errp, errno,
+ "RDMA ERROR: failed to get cm event");
 }
 }
 if (ret < 0) {
-/*
- * FIXME perror() is wrong, because
- * qemu_get_cm_event_timeout() can fail without setting errno.
- * Will go away later in this series.
- */
-perror("rdma_get_cm_event after rdma_connect");
 goto err_rdma_source_connect;
 }
 
 if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) {
-error_report("rdma_get_cm_event != EVENT_ESTABLISHED after 
rdma_connect");
 error_setg(errp, "RDMA ERROR: connecting to destination!");
 rdma_ack_cm_event(cm_event);
 goto err_rdma_source_connect;
-- 
2.41.0

[PULL 46/65] migration/rdma: Convert qemu_rdma_write_flush() to Error

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qio_channel_rdma_writev() violates this principle: it calls
error_report() via qemu_rdma_write_flush().  I elected not to
investigate how callers handle the error, i.e. precise impact is not
known.

Clean this up by converting qemu_rdma_write_flush() to Error.

Necessitates setting an error when qemu_rdma_write_one() failed.
Since this error will go away later in this series, simply use "FIXME
temporary error message" there.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-40-arm...@redhat.com>
---
 migration/rdma.c | 19 +--
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 189932097e..1a74c6d955 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2283,7 +2283,7 @@ retry:
  * We support sending out multiple chunks at the same time.
  * Not all of them need to get signaled in the completion queue.
  */
-static int qemu_rdma_write_flush(RDMAContext *rdma)
+static int qemu_rdma_write_flush(RDMAContext *rdma, Error **errp)
 {
 int ret;
 
@@ -2295,6 +2295,7 @@ static int qemu_rdma_write_flush(RDMAContext *rdma)
 rdma->current_index, rdma->current_addr, rdma->current_length);
 
 if (ret < 0) {
+error_setg(errp, "FIXME temporary error message");
 return -1;
 }
 
@@ -2368,6 +2369,7 @@ static int qemu_rdma_write(RDMAContext *rdma,
uint64_t block_offset, uint64_t offset,
uint64_t len)
 {
+Error *err = NULL;
 uint64_t current_addr = block_offset + offset;
 uint64_t index = rdma->current_index;
 uint64_t chunk = rdma->current_chunk;
@@ -2375,8 +2377,9 @@ static int qemu_rdma_write(RDMAContext *rdma,
 
 /* If we cannot merge it, we flush the current buffer first. */
 if (!qemu_rdma_buffer_mergeable(rdma, current_addr, len)) {
-ret = qemu_rdma_write_flush(rdma);
+ret = qemu_rdma_write_flush(rdma, &err);
 if (ret < 0) {
+error_report_err(err);
 return -1;
 }
 rdma->current_length = 0;
@@ -2393,7 +2396,10 @@ static int qemu_rdma_write(RDMAContext *rdma,
 
 /* flush it if buffer is too large */
 if (rdma->current_length >= RDMA_MERGE_MAX) {
-return qemu_rdma_write_flush(rdma);
+if (qemu_rdma_write_flush(rdma, &err) < 0) {
+error_report_err(err);
+return -1;
+}
 }
 
 return 0;
@@ -2857,10 +2863,9 @@ static ssize_t qio_channel_rdma_writev(QIOChannel *ioc,
  * Push out any writes that
  * we're queued up for VM's ram.
  */
-ret = qemu_rdma_write_flush(rdma);
+ret = qemu_rdma_write_flush(rdma, errp);
 if (ret < 0) {
 rdma->errored = true;
-error_setg(errp, "qemu_rdma_write_flush failed");
 return -1;
 }
 
@@ -3002,9 +3007,11 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
  */
 static int qemu_rdma_drain_cq(RDMAContext *rdma)
 {
+Error *err = NULL;
 int ret;
 
-if (qemu_rdma_write_flush(rdma) < 0) {
+if (qemu_rdma_write_flush(rdma, &err) < 0) {
+error_report_err(err);
 return -1;
 }
 
-- 
2.41.0

[PULL 55/65] migration/rdma: Don't report received completion events as error

[Juan Quintela]

From: Markus Armbruster 

When qemu_rdma_wait_comp_channel() receives an event from the
completion channel, it reports an error "receive cm event while wait
comp channel,cm event is T", where T is the numeric event type.
However, the function fails only when T is a disconnect or device
removal.  Events other than these two are not actually an error, and
reporting them as an error is wrong.  If we need to report them to the
user, we should use something else, and what to use depends on why we
need to report them to the user.

For now, report this error only when the function actually fails.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-49-arm...@redhat.com>
---
 migration/rdma.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 6a8ff8fa96..b5bb47108e 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1582,11 +1582,11 @@ static int qemu_rdma_wait_comp_channel(RDMAContext 
*rdma,
 return -1;
 }
 
-error_report("receive cm event while wait comp channel,"
- "cm event is %d", cm_event->event);
 if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED ||
 cm_event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) {
 rdma_ack_cm_event(cm_event);
+error_report("receive cm event while wait comp 
channel,"
+ "cm event is %d", cm_event->event);
 return -1;
 }
 rdma_ack_cm_event(cm_event);
-- 
2.41.0

[PULL 12/65] migration/rdma: Consistently use uint64_t for work request IDs

[Juan Quintela]

From: Markus Armbruster 

We use int instead of uint64_t in a few places.  Change them to
uint64_t.

This cleans up a comparison of signed qemu_rdma_block_for_wrid()
parameter @wrid_requested with unsigned @wr_id.  Harmless, because the
actual arguments are non-negative enumeration constants.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-6-arm...@redhat.com>
---
 migration/rdma.c   | 7 ---
 migration/trace-events | 8 
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 8f297c9e46..d1e727f30b 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1599,13 +1599,13 @@ static int qemu_rdma_wait_comp_channel(RDMAContext 
*rdma,
 return rdma->error_state;
 }
 
-static struct ibv_comp_channel *to_channel(RDMAContext *rdma, int wrid)
+static struct ibv_comp_channel *to_channel(RDMAContext *rdma, uint64_t wrid)
 {
 return wrid < RDMA_WRID_RECV_CONTROL ? rdma->send_comp_channel :
rdma->recv_comp_channel;
 }
 
-static struct ibv_cq *to_cq(RDMAContext *rdma, int wrid)
+static struct ibv_cq *to_cq(RDMAContext *rdma, uint64_t wrid)
 {
 return wrid < RDMA_WRID_RECV_CONTROL ? rdma->send_cq : rdma->recv_cq;
 }
@@ -1623,7 +1623,8 @@ static struct ibv_cq *to_cq(RDMAContext *rdma, int wrid)
  * completions only need to be recorded, but do not actually
  * need further processing.
  */
-static int qemu_rdma_block_for_wrid(RDMAContext *rdma, int wrid_requested,
+static int qemu_rdma_block_for_wrid(RDMAContext *rdma,
+uint64_t wrid_requested,
 uint32_t *byte_len)
 {
 int num_cq_events = 0, ret = 0;
diff --git a/migration/trace-events b/migration/trace-events
index 19f9ee7c6d..6a50994402 100644
--- a/migration/trace-events
+++ b/migration/trace-events
@@ -208,7 +208,7 @@ qemu_rdma_accept_incoming_migration(void) ""
 qemu_rdma_accept_incoming_migration_accepted(void) ""
 qemu_rdma_accept_pin_state(bool pin) "%d"
 qemu_rdma_accept_pin_verbsc(void *verbs) "Verbs context after listen: %p"
-qemu_rdma_block_for_wrid_miss(int wcomp, uint64_t req) "A Wanted wrid %d but 
got %" PRIu64
+qemu_rdma_block_for_wrid_miss(uint64_t wcomp, uint64_t req) "A Wanted wrid %" 
PRIu64 " but got %" PRIu64
 qemu_rdma_cleanup_disconnect(void) ""
 qemu_rdma_close(void) ""
 qemu_rdma_connect_pin_all_requested(void) ""
@@ -222,9 +222,9 @@ qemu_rdma_exchange_send_waiting(const char *desc) "Waiting 
for response %s"
 qemu_rdma_exchange_send_received(const char *desc) "Response %s received."
 qemu_rdma_fill(size_t control_len, size_t size) "RDMA %zd of %zd bytes already 
in buffer"
 qemu_rdma_init_ram_blocks(int blocks) "Allocated %d local ram block structures"
-qemu_rdma_poll_recv(int64_t comp, int64_t id, int sent) "completion %" PRId64 
" received (%" PRId64 ") left %d"
-qemu_rdma_poll_write(int64_t comp, int left, uint64_t block, uint64_t chunk, 
void *local, void *remote) "completions %" PRId64 " left %d, block %" PRIu64 ", 
chunk: %" PRIu64 " %p %p"
-qemu_rdma_poll_other(int64_t comp, int left) "other completion %" PRId64 " 
received left %d"
+qemu_rdma_poll_recv(uint64_t comp, int64_t id, int sent) "completion %" PRIu64 
" received (%" PRId64 ") left %d"
+qemu_rdma_poll_write(uint64_t comp, int left, uint64_t block, uint64_t chunk, 
void *local, void *remote) "completions %" PRIu64 " left %d, block %" PRIu64 ", 
chunk: %" PRIu64 " %p %p"
+qemu_rdma_poll_other(uint64_t comp, int left) "other completion %" PRIu64 " 
received left %d"
 qemu_rdma_post_send_control(const char *desc) "CONTROL: sending %s.."
 qemu_rdma_register_and_get_keys(uint64_t len, void *start) "Registering %" 
PRIu64 " bytes @ %p"
 qemu_rdma_register_odp_mr(const char *name) "Try to register On-Demand Paging 
memory region: %s"
-- 
2.41.0

[PULL 19/65] migration/rdma: Drop rdma_add_block() error handling

[Juan Quintela]

From: Markus Armbruster 

rdma_add_block() can't fail.  Return void, and drop the unreachable
error handling.

Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-13-arm...@redhat.com>
---
 migration/rdma.c | 30 +-
 1 file changed, 9 insertions(+), 21 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index c858d3fbe4..466725dbf0 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -559,9 +559,9 @@ static inline uint8_t *ram_chunk_end(const RDMALocalBlock 
*rdma_ram_block,
 return result;
 }
 
-static int rdma_add_block(RDMAContext *rdma, const char *block_name,
- void *host_addr,
- ram_addr_t block_offset, uint64_t length)
+static void rdma_add_block(RDMAContext *rdma, const char *block_name,
+   void *host_addr,
+   ram_addr_t block_offset, uint64_t length)
 {
 RDMALocalBlocks *local = &rdma->local_ram_blocks;
 RDMALocalBlock *block;
@@ -615,8 +615,6 @@ static int rdma_add_block(RDMAContext *rdma, const char 
*block_name,
  block->nb_chunks);
 
 local->nb_blocks++;
-
-return 0;
 }
 
 /*
@@ -630,7 +628,8 @@ static int qemu_rdma_init_one_block(RAMBlock *rb, void 
*opaque)
 void *host_addr = qemu_ram_get_host_addr(rb);
 ram_addr_t block_offset = qemu_ram_get_offset(rb);
 ram_addr_t length = qemu_ram_get_used_length(rb);
-return rdma_add_block(opaque, block_name, host_addr, block_offset, length);
+rdma_add_block(opaque, block_name, host_addr, block_offset, length);
+return 0;
 }
 
 /*
@@ -638,7 +637,7 @@ static int qemu_rdma_init_one_block(RAMBlock *rb, void 
*opaque)
  * identify chunk boundaries inside each RAMBlock and also be referenced
  * during dynamic page registration.
  */
-static int qemu_rdma_init_ram_blocks(RDMAContext *rdma)
+static void qemu_rdma_init_ram_blocks(RDMAContext *rdma)
 {
 RDMALocalBlocks *local = &rdma->local_ram_blocks;
 int ret;
@@ -646,14 +645,11 @@ static int qemu_rdma_init_ram_blocks(RDMAContext *rdma)
 assert(rdma->blockmap == NULL);
 memset(local, 0, sizeof *local);
 ret = foreach_not_ignored_block(qemu_rdma_init_one_block, rdma);
-if (ret) {
-return ret;
-}
+assert(!ret);
 trace_qemu_rdma_init_ram_blocks(local->nb_blocks);
 rdma->dest_blocks = g_new0(RDMADestBlock,
rdma->local_ram_blocks.nb_blocks);
 local->init = true;
-return 0;
 }
 
 /*
@@ -2491,11 +2487,7 @@ static int qemu_rdma_source_init(RDMAContext *rdma, bool 
pin_all, Error **errp)
 goto err_rdma_source_init;
 }
 
-ret = qemu_rdma_init_ram_blocks(rdma);
-if (ret) {
-ERROR(errp, "rdma migration: error initializing ram blocks!");
-goto err_rdma_source_init;
-}
+qemu_rdma_init_ram_blocks(rdma);
 
 /* Build the hash that maps from offset to RAMBlock */
 rdma->blockmap = g_hash_table_new(g_direct_hash, g_direct_equal);
@@ -3438,11 +3430,7 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 goto err_rdma_dest_wait;
 }
 
-ret = qemu_rdma_init_ram_blocks(rdma);
-if (ret) {
-error_report("rdma migration: error initializing ram blocks!");
-goto err_rdma_dest_wait;
-}
+qemu_rdma_init_ram_blocks(rdma);
 
 for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
 ret = qemu_rdma_reg_control(rdma, idx);
-- 
2.41.0

[PULL 06/65] tests/qtest: migration: Add support for negative testing of qmp_migrate

[Juan Quintela]

From: Fabiano Rosas 

There is currently no way to write a test for errors that happened in
qmp_migrate before the migration has started.

Add a version of qmp_migrate that ensures an error happens. To make
use of it a test needs to set MigrateCommon.result as
MIG_TEST_QMP_ERROR.

Reviewed-by: Peter Xu 
Reviewed-by: Juan Quintela 
Signed-off-by: Fabiano Rosas 
Signed-off-by: Juan Quintela 
Message-ID: <20230712190742.22294-6-faro...@suse.de>
---
 tests/qtest/libqtest.h  | 28 
 tests/qtest/migration-helpers.h |  3 +++
 tests/qtest/libqtest.c  | 33 +
 tests/qtest/migration-helpers.c | 20 
 tests/qtest/migration-test.c| 16 
 5 files changed, 96 insertions(+), 4 deletions(-)

diff --git a/tests/qtest/libqtest.h b/tests/qtest/libqtest.h
index e53e350e3a..5fe3d13466 100644
--- a/tests/qtest/libqtest.h
+++ b/tests/qtest/libqtest.h
@@ -810,6 +810,34 @@ void qtest_vqmp_fds_assert_success(QTestState *qts, int 
*fds, size_t nfds,
 G_GNUC_PRINTF(4, 0);
 #endif /* !_WIN32 */
 
+/**
+ * qtest_qmp_assert_failure_ref:
+ * @qts: QTestState instance to operate on
+ * @fmt: QMP message to send to qemu, formatted like
+ * qobject_from_jsonf_nofail().  See parse_interpolation() for what's
+ * supported after '%'.
+ *
+ * Sends a QMP message to QEMU, asserts that an 'error' key is present in
+ * the response, and returns the response.
+ */
+QDict *qtest_qmp_assert_failure_ref(QTestState *qts, const char *fmt, ...)
+G_GNUC_PRINTF(2, 3);
+
+/**
+ * qtest_vqmp_assert_failure_ref:
+ * @qts: QTestState instance to operate on
+ * @fmt: QMP message to send to qemu, formatted like
+ * qobject_from_jsonf_nofail().  See parse_interpolation() for what's
+ * supported after '%'.
+ * @args: variable arguments for @fmt
+ *
+ * Sends a QMP message to QEMU, asserts that an 'error' key is present in
+ * the response, and returns the response.
+ */
+QDict *qtest_vqmp_assert_failure_ref(QTestState *qts,
+ const char *fmt, va_list args)
+G_GNUC_PRINTF(2, 0);
+
 /**
  * qtest_qmp_assert_success_ref:
  * @qts: QTestState instance to operate on
diff --git a/tests/qtest/migration-helpers.h b/tests/qtest/migration-helpers.h
index 57d295a4fe..4f51d0f8bc 100644
--- a/tests/qtest/migration-helpers.h
+++ b/tests/qtest/migration-helpers.h
@@ -27,6 +27,9 @@ G_GNUC_PRINTF(3, 4)
 void migrate_incoming_qmp(QTestState *who, const char *uri,
   const char *fmt, ...);
 
+G_GNUC_PRINTF(3, 4)
+void migrate_qmp_fail(QTestState *who, const char *uri, const char *fmt, ...);
+
 void migrate_set_capability(QTestState *who, const char *capability,
 bool value);
 
diff --git a/tests/qtest/libqtest.c b/tests/qtest/libqtest.c
index 3f94a4f477..dc7a55634c 100644
--- a/tests/qtest/libqtest.c
+++ b/tests/qtest/libqtest.c
@@ -1259,6 +1259,28 @@ void qtest_memset(QTestState *s, uint64_t addr, uint8_t 
pattern, size_t size)
 qtest_rsp(s);
 }
 
+QDict *qtest_vqmp_assert_failure_ref(QTestState *qts,
+ const char *fmt, va_list args)
+{
+QDict *response;
+QDict *ret;
+
+response = qtest_vqmp(qts, fmt, args);
+
+g_assert(response);
+if (!qdict_haskey(response, "error")) {
+g_autoptr(GString) s = qobject_to_json_pretty(QOBJECT(response), true);
+g_test_message("%s", s->str);
+}
+g_assert(qdict_haskey(response, "error"));
+g_assert(!qdict_haskey(response, "return"));
+ret = qdict_get_qdict(response, "error");
+qobject_ref(ret);
+qobject_unref(response);
+
+return ret;
+}
+
 QDict *qtest_vqmp_assert_success_ref(QTestState *qts,
  const char *fmt, va_list args)
 {
@@ -1321,6 +1343,17 @@ void qtest_vqmp_fds_assert_success(QTestState *qts, int 
*fds, size_t nfds,
 }
 #endif /* !_WIN32 */
 
+QDict *qtest_qmp_assert_failure_ref(QTestState *qts, const char *fmt, ...)
+{
+QDict *response;
+va_list ap;
+
+va_start(ap, fmt);
+response = qtest_vqmp_assert_failure_ref(qts, fmt, ap);
+va_end(ap);
+return response;
+}
+
 QDict *qtest_qmp_assert_success_ref(QTestState *qts, const char *fmt, ...)
 {
 QDict *response;
diff --git a/tests/qtest/migration-helpers.c b/tests/qtest/migration-helpers.c
index 08f5ee1179..0c185db450 100644
--- a/tests/qtest/migration-helpers.c
+++ b/tests/qtest/migration-helpers.c
@@ -49,6 +49,26 @@ bool migrate_watch_for_resume(QTestState *who, const char 
*name,
 return false;
 }
 
+void migrate_qmp_fail(QTestState *who, const char *uri, const char *fmt, ...)
+{
+va_list ap;
+QDict *args, *err;
+
+va_start(ap, fmt);
+args = qdict_from_vjsonf_nofail(fmt, ap);
+va_end(ap);
+
+g_assert(!qdict_haskey(args, "uri"));
+qdict_put_str(args, "uri", uri);
+
+err = qtest_qmp_assert_failure_ref(
+who, "{ 'execute': 'migrate', 'arguments': %p}", args);
+
+g_assert(qdi

[PULL 54/65] migration/rdma: Silence qemu_rdma_reg_control()

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_source_init() and qemu_rdma_accept() violate this principle:
they call error_report() via qemu_rdma_reg_control().  I elected not
to investigate how callers handle the error, i.e. precise impact is
not known.

Clean this up by dropping the error reporting from
qemu_rdma_reg_control().  I believe the callers' error reports
suffice.  If they don't, we need to convert to Error instead.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-48-arm...@redhat.com>
---
 migration/rdma.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 8e1e8c4d47..6a8ff8fa96 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1360,7 +1360,6 @@ static int qemu_rdma_reg_control(RDMAContext *rdma, int 
idx)
 rdma->total_registrations++;
 return 0;
 }
-error_report("qemu_rdma_reg_control failed");
 return -1;
 }
 
-- 
2.41.0

[PULL 13/65] migration/rdma: Fix unwanted integer truncation

[Juan Quintela]

From: Markus Armbruster 

qio_channel_rdma_readv() assigns the size_t value of qemu_rdma_fill()
to an int variable before it adds it to @done / subtracts it from
@want, both size_t.  Truncation when qemu_rdma_fill() copies more than
INT_MAX bytes.  Seems vanishingly unlikely, but needs fixing all the
same.

Fixes: 6ddd2d76ca6f (migration: convert RDMA to use QIOChannel interface)
Signed-off-by: Markus Armbruster 
Reviewed-by: Fabiano Rosas 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-7-arm...@redhat.com>
---
 migration/rdma.c | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index d1e727f30b..ff8e475f59 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2871,7 +2871,7 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
 RDMAControlHeader head;
 int ret = 0;
 ssize_t i;
-size_t done = 0;
+size_t done = 0, len;
 
 RCU_READ_LOCK_GUARD();
 rdma = qatomic_rcu_read(&rioc->rdmain);
@@ -2892,9 +2892,9 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
  * were given and dish out the bytes until we run
  * out of bytes.
  */
-ret = qemu_rdma_fill(rdma, data, want, 0);
-done += ret;
-want -= ret;
+len = qemu_rdma_fill(rdma, data, want, 0);
+done += len;
+want -= len;
 /* Got what we needed, so go to next iovec */
 if (want == 0) {
 continue;
@@ -2921,9 +2921,9 @@ static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
 /*
  * SEND was received with new bytes, now try again.
  */
-ret = qemu_rdma_fill(rdma, data, want, 0);
-done += ret;
-want -= ret;
+len = qemu_rdma_fill(rdma, data, want, 0);
+done += len;
+want -= len;
 
 /* Still didn't get enough, so lets just return */
 if (want) {
-- 
2.41.0

[PULL 48/65] migration/rdma: Convert qemu_rdma_write() to Error

[Juan Quintela]

From: Markus Armbruster 

Just for consistency with qemu_rdma_write_one() and
qemu_rdma_write_flush(), and for slightly simpler code.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-42-arm...@redhat.com>
---
 migration/rdma.c | 16 ++--
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 369d30c895..9f45f6a91d 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -2359,9 +2359,8 @@ static inline bool qemu_rdma_buffer_mergeable(RDMAContext 
*rdma,
  */
 static int qemu_rdma_write(RDMAContext *rdma,
uint64_t block_offset, uint64_t offset,
-   uint64_t len)
+   uint64_t len, Error **errp)
 {
-Error *err = NULL;
 uint64_t current_addr = block_offset + offset;
 uint64_t index = rdma->current_index;
 uint64_t chunk = rdma->current_chunk;
@@ -2369,9 +2368,8 @@ static int qemu_rdma_write(RDMAContext *rdma,
 
 /* If we cannot merge it, we flush the current buffer first. */
 if (!qemu_rdma_buffer_mergeable(rdma, current_addr, len)) {
-ret = qemu_rdma_write_flush(rdma, &err);
+ret = qemu_rdma_write_flush(rdma, errp);
 if (ret < 0) {
-error_report_err(err);
 return -1;
 }
 rdma->current_length = 0;
@@ -2388,10 +2386,7 @@ static int qemu_rdma_write(RDMAContext *rdma,
 
 /* flush it if buffer is too large */
 if (rdma->current_length >= RDMA_MERGE_MAX) {
-if (qemu_rdma_write_flush(rdma, &err) < 0) {
-error_report_err(err);
-return -1;
-}
+return qemu_rdma_write_flush(rdma, errp);
 }
 
 return 0;
@@ -3290,6 +3285,7 @@ static int qemu_rdma_save_page(QEMUFile *f, ram_addr_t 
block_offset,
ram_addr_t offset, size_t size)
 {
 QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(qemu_file_get_ioc(f));
+Error *err = NULL;
 RDMAContext *rdma;
 int ret;
 
@@ -3315,9 +3311,9 @@ static int qemu_rdma_save_page(QEMUFile *f, ram_addr_t 
block_offset,
  * is full, or the page doesn't belong to the current chunk,
  * an actual RDMA write will occur and a new chunk will be formed.
  */
-ret = qemu_rdma_write(rdma, block_offset, offset, size);
+ret = qemu_rdma_write(rdma, block_offset, offset, size, &err);
 if (ret < 0) {
-error_report("rdma migration: write error");
+error_report_err(err);
 goto err;
 }
 
-- 
2.41.0

[PULL 57/65] migration/rdma: Silence qemu_rdma_register_and_get_keys()

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_write_one() violates this principle: it reports errors to
stderr via qemu_rdma_register_and_get_keys().  I elected not to
investigate how callers handle the error, i.e. precise impact is not
known.

Clean this up: silence qemu_rdma_register_and_get_keys().  I believe
the caller's error reports suffice.  If they don't, we need to convert
to Error instead.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-51-arm...@redhat.com>
---
 migration/rdma.c | 9 -
 1 file changed, 9 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 459dcb002e..025523bf70 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1325,15 +1325,6 @@ static int qemu_rdma_register_and_get_keys(RDMAContext 
*rdma,
 }
 }
 if (!block->pmr[chunk]) {
-perror("Failed to register chunk!");
-fprintf(stderr, "Chunk details: block: %d chunk index %d"
-" start %" PRIuPTR " end %" PRIuPTR
-" host %" PRIuPTR
-" local %" PRIuPTR " registrations: %d\n",
-block->index, chunk, (uintptr_t)chunk_start,
-(uintptr_t)chunk_end, host_addr,
-(uintptr_t)block->local_host_addr,
-rdma->total_registrations);
 return -1;
 }
 rdma->total_registrations++;
-- 
2.41.0

[PULL 50/65] migration/rdma: Convert qemu_rdma_post_recv_control() to Error

[Juan Quintela]

From: Markus Armbruster 

Just for symmetry with qemu_rdma_post_send_control().  Error messages
lose detail I consider of no use to users.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-44-arm...@redhat.com>
---
 migration/rdma.c | 22 ++
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index aeb0a8921e..41ea2edcda 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1799,7 +1799,8 @@ static int qemu_rdma_post_send_control(RDMAContext *rdma, 
uint8_t *buf,
  * Post a RECV work request in anticipation of some future receipt
  * of data on the control channel.
  */
-static int qemu_rdma_post_recv_control(RDMAContext *rdma, int idx)
+static int qemu_rdma_post_recv_control(RDMAContext *rdma, int idx,
+   Error **errp)
 {
 struct ibv_recv_wr *bad_wr;
 struct ibv_sge sge = {
@@ -1816,6 +1817,7 @@ static int qemu_rdma_post_recv_control(RDMAContext *rdma, 
int idx)
 
 
 if (ibv_post_recv(rdma->qp, &recv_wr, &bad_wr)) {
+error_setg(errp, "error posting control recv");
 return -1;
 }
 
@@ -1926,10 +1928,8 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
  * If the user is expecting a response, post a WR in anticipation of it.
  */
 if (resp) {
-ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA);
+ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA, errp);
 if (ret < 0) {
-error_setg(errp, "rdma migration: error posting"
-" extra control recv for anticipated result!");
 return -1;
 }
 }
@@ -1937,9 +1937,8 @@ static int qemu_rdma_exchange_send(RDMAContext *rdma, 
RDMAControlHeader *head,
 /*
  * Post a WR to replace the one we just consumed for the READY message.
  */
-ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, errp);
 if (ret < 0) {
-error_setg(errp, "rdma migration: error posting first control recv!");
 return -1;
 }
 
@@ -2023,9 +2022,8 @@ static int qemu_rdma_exchange_recv(RDMAContext *rdma, 
RDMAControlHeader *head,
 /*
  * Post a new RECV work request to replace the one we just consumed.
  */
-ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, errp);
 if (ret < 0) {
-error_setg(errp, "rdma migration: error posting second control recv!");
 return -1;
 }
 
@@ -2608,9 +2606,8 @@ static int qemu_rdma_connect(RDMAContext *rdma, bool 
return_path,
 
 caps_to_network(&cap);
 
-ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, errp);
 if (ret < 0) {
-error_setg(errp, "RDMA ERROR: posting second control recv");
 goto err_rdma_source_connect;
 }
 
@@ -3402,6 +3399,7 @@ static void rdma_cm_poll_handler(void *opaque)
 
 static int qemu_rdma_accept(RDMAContext *rdma)
 {
+Error *err = NULL;
 RDMACapabilities cap;
 struct rdma_conn_param conn_param = {
 .responder_resources = 2,
@@ -3538,9 +3536,9 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 rdma_ack_cm_event(cm_event);
 rdma->connected = true;
 
-ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY, &err);
 if (ret < 0) {
-error_report("rdma migration: error posting second control recv");
+error_report_err(err);
 goto err_rdma_dest_wait;
 }
 
-- 
2.41.0

[PULL 51/65] migration/rdma: Convert qemu_rdma_alloc_pd_cq() to Error

[Juan Quintela]

From: Markus Armbruster 

Functions that use an Error **errp parameter to return errors should
not also report them to the user, because reporting is the caller's
job.  When the caller does, the error is reported twice.  When it
doesn't (because it recovered from the error), there is no error to
report, i.e. the report is bogus.

qemu_rdma_source_init() violates this principle: it calls
error_report() via qemu_rdma_alloc_pd_cq().  I elected not to
investigate how callers handle the error, i.e. precise impact is not
known.

Clean this up by converting qemu_rdma_alloc_pd_cq() to Error.

The conversion loses a piece of advice on one of two failure paths:

Your mlock() limits may be too low. Please check $ ulimit -a # and search 
for 'ulimit -l' in the output

Not worth retaining.

Signed-off-by: Markus Armbruster 
Reviewed-by: Li Zhijian 
Reviewed-by: Juan Quintela 
Signed-off-by: Juan Quintela 
Message-ID: <20230928132019.2544702-45-arm...@redhat.com>
---
 migration/rdma.c | 22 +-
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/migration/rdma.c b/migration/rdma.c
index 41ea2edcda..ee9221d5d2 100644
--- a/migration/rdma.c
+++ b/migration/rdma.c
@@ -1052,19 +1052,19 @@ err_resolve_create_id:
 /*
  * Create protection domain and completion queues
  */
-static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma)
+static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma, Error **errp)
 {
 /* allocate pd */
 rdma->pd = ibv_alloc_pd(rdma->verbs);
 if (!rdma->pd) {
-error_report("failed to allocate protection domain");
+error_setg(errp, "failed to allocate protection domain");
 return -1;
 }
 
 /* create receive completion channel */
 rdma->recv_comp_channel = ibv_create_comp_channel(rdma->verbs);
 if (!rdma->recv_comp_channel) {
-error_report("failed to allocate receive completion channel");
+error_setg(errp, "failed to allocate receive completion channel");
 goto err_alloc_pd_cq;
 }
 
@@ -1074,21 +1074,21 @@ static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma)
 rdma->recv_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3),
   NULL, rdma->recv_comp_channel, 0);
 if (!rdma->recv_cq) {
-error_report("failed to allocate receive completion queue");
+error_setg(errp, "failed to allocate receive completion queue");
 goto err_alloc_pd_cq;
 }
 
 /* create send completion channel */
 rdma->send_comp_channel = ibv_create_comp_channel(rdma->verbs);
 if (!rdma->send_comp_channel) {
-error_report("failed to allocate send completion channel");
+error_setg(errp, "failed to allocate send completion channel");
 goto err_alloc_pd_cq;
 }
 
 rdma->send_cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3),
   NULL, rdma->send_comp_channel, 0);
 if (!rdma->send_cq) {
-error_report("failed to allocate send completion queue");
+error_setg(errp, "failed to allocate send completion queue");
 goto err_alloc_pd_cq;
 }
 
@@ -2503,12 +2503,8 @@ static int qemu_rdma_source_init(RDMAContext *rdma, bool 
pin_all, Error **errp)
 goto err_rdma_source_init;
 }
 
-ret = qemu_rdma_alloc_pd_cq(rdma);
+ret = qemu_rdma_alloc_pd_cq(rdma, errp);
 if (ret < 0) {
-error_setg(errp, "RDMA ERROR: "
-   "rdma migration: error allocating pd and cq! Your mlock()"
-   " limits may be too low. Please check $ ulimit -a # and "
-   "search for 'ulimit -l' in the output");
 goto err_rdma_source_init;
 }
 
@@ -3482,9 +3478,9 @@ static int qemu_rdma_accept(RDMAContext *rdma)
 
 qemu_rdma_dump_id("dest_init", verbs);
 
-ret = qemu_rdma_alloc_pd_cq(rdma);
+ret = qemu_rdma_alloc_pd_cq(rdma, &err);
 if (ret < 0) {
-error_report("rdma migration: error allocating pd and cq!");
+error_report_err(err);
 goto err_rdma_dest_wait;
 }
 
-- 
2.41.0

Re: [PATCH 2/3] target/riscv: Support discontinuous PMU counters

[Rob Bradford]

On Mon, 2023-10-09 at 11:00 -0700, Atish Kumar Patra wrote:
> On Sun, Oct 8, 2023 at 5:58 PM Alistair Francis
>  wrote:
> > 
> > On Wed, Oct 4, 2023 at 7:36 PM Rob Bradford
> >  wrote:
> > > 
> > > Hi Atish,
> > > 
> > > On Tue, 2023-10-03 at 13:25 -0700, Atish Kumar Patra wrote:
> > > > On Tue, Oct 3, 2023 at 5:51 AM Rob Bradford
> > > > 
> > > > wrote:
> > > > > 
> > > > > There is no requirement that the enabled counters in the
> > > > > platform
> > > > > are
> > > > > continuously numbered. Add a "pmu-mask" property that, if
> > > > > specified, can
> > > > > be used to specify the enabled PMUs. In order to avoid
> > > > > ambiguity if
> > > > > "pmu-mask" is specified then "pmu-num" must also match the
> > > > > number
> > > > > of
> > > > > bits set in the mask.
> > > > > 
> > > > > Signed-off-by: Rob Bradford 
> > > > > ---
> > > > >  target/riscv/cpu.c |  1 +
> > > > >  target/riscv/cpu_cfg.h |  1 +
> > > > >  target/riscv/pmu.c | 15 +--
> > > > >  3 files changed, 15 insertions(+), 2 deletions(-)
> > > > > 
> > > > > diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c
> > > > > index 9d79c20c1a..b89b006a76 100644
> > > > > --- a/target/riscv/cpu.c
> > > > > +++ b/target/riscv/cpu.c
> > > > > @@ -1817,6 +1817,7 @@ static void
> > > > > riscv_cpu_add_misa_properties(Object *cpu_obj)
> > > > >  static Property riscv_cpu_extensions[] = {
> > > > >  /* Defaults for standard extensions */
> > > > >  DEFINE_PROP_UINT8("pmu-num", RISCVCPU, cfg.pmu_num, 16),
> > > > > +    DEFINE_PROP_UINT32("pmu-mask", RISCVCPU, cfg.pmu_mask,
> > > > > 0),
> > > > >  DEFINE_PROP_BOOL("sscofpmf", RISCVCPU, cfg.ext_sscofpmf,
> > > > > false),
> > > > >  DEFINE_PROP_BOOL("Zifencei", RISCVCPU, cfg.ext_ifencei,
> > > > > true),
> > > > >  DEFINE_PROP_BOOL("Zicsr", RISCVCPU, cfg.ext_icsr, true),
> > > > > diff --git a/target/riscv/cpu_cfg.h b/target/riscv/cpu_cfg.h
> > > > > index 0e6a0f245c..40f7d970bc 100644
> > > > > --- a/target/riscv/cpu_cfg.h
> > > > > +++ b/target/riscv/cpu_cfg.h
> > > > > @@ -124,6 +124,7 @@ struct RISCVCPUConfig {
> > > > >  bool ext_XVentanaCondOps;
> > > > > 
> > > > >  uint8_t pmu_num;
> > > > > +    uint32_t pmu_mask;
> > > > >  char *priv_spec;
> > > > >  char *user_spec;
> > > > >  char *bext_spec;
> > > > > diff --git a/target/riscv/pmu.c b/target/riscv/pmu.c
> > > > > index 13801ccb78..f97e25a1f6 100644
> > > > > --- a/target/riscv/pmu.c
> > > > > +++ b/target/riscv/pmu.c
> > > > > @@ -437,6 +437,13 @@ int riscv_pmu_setup_timer(CPURISCVState
> > > > > *env,
> > > > > uint64_t value, uint32_t ctr_idx)
> > > > >  void riscv_pmu_init(RISCVCPU *cpu, Error **errp)
> > > > >  {
> > > > >  uint8_t pmu_num = cpu->cfg.pmu_num;
> > > > > +    uint32_t pmu_mask = cpu->cfg.pmu_mask;
> > > > > +
> > > > > +    if (pmu_mask && ctpop32(pmu_mask) != pmu_num) {
> > > > > +    error_setg(errp, "Mismatch between number of enabled
> > > > > counters in "
> > > > > + "\"pmu-mask\" and \"pmu-num\"");
> > > > > +    return;
> > > > > +    }
> > > > > 
> > > > 
> > > > Is that necessary for the default case? I am thinking of
> > > > marking
> > > > pmu-num as deprecated and pmu-mask
> > > > as the preferred way of doing things as it is more flexible.
> > > > There is
> > > > no real benefit carrying both.
> > > > The default pmu-mask value will change in that case.
> > > > We can just overwrite pmu-num with ctpop32(pmu_mask) if pmu-
> > > > mask is
> > > > available. Thoughts ?
> > > > 
> > > 
> > > I agree it makes sense to me that there is only one way for the
> > > user to
> > > adjust the PMU count. However i'm not sure how we can handle the
> > > transition if we choose to deprecate "pmu-num".
> > > 
> > > If we change the default "pmu-mask" to MAKE_32BIT_MASK(3, 16)
> > > then that
> > > value in the config will always be set - you propose that we
> > > overwrite
> > > "pmu-num" with the popcount of that property. But that will break
> > > if
> > 
> > Couldn't we deprecate "pmu-num" and then throw an error if both are
> > set? Then we can migrate away from "pmu-num"
> > 
> 
> Yeah. pmu-num should be only available as a command line property and
> marked deprecated.
> If only pmu-num is set, it gets converted to a mask and throws a
> warning
> that this is a deprecated property.

Is there a way to know the property has been set by the user? I
couldn't see anything in the API - do we just have to assume that if
the value is not the default then it has been changed by the user?

Cheers,

Rob

> If only the pmu-mask is set, nothing additional is needed. These
> patches are sufficient.
> If nothing is set, the pmu-mask will be set to MAKE_32BIT_MASK(3,
> 16).
> If a CPU init code uses pmu-num, we should change it to mask. The
> upstream code
> doesn't have any other usage. Any downstream user will have to move
> away from pmu-num
> once this series is merged.
> 
> > Alistair
> > 
> > > the user has an existing setup that