Could anybody review it? On 12/21/2012 05:46 PM, Satoru Moriya wrote: > Changelog > v1 -> v2 > - add RFC tag again > - change semantics as follows > - set event threads' priority to maxprio > - set vcpu threads' priority to maxprio - 1 > - isolate all the posix stuff and put them into os_prioritize() in > os-posix.c/qemu-os-win32.h to avoid breaking win32 build > - introduce qemu_init_realtime(), qemu_realtime_is_enable and > qemu_realtime_get_parameters() and struct QemuRealtimeInfo to > keep realtime option and remove related global variables in vl.c > - add other benchmark(qpid-latency-test) result > > We have some plans to migrate old enterprise/control systems which > require low latency (msec order) to kvm virtualized environment. In > order to satify the requirements, this patch adds realtime option to qemu: > > -realtime maxprio=<prio>,policy=<pol> > > This option change the scheduling policy and priority to realtime one > (event threads: maxprio, vcpu threads: maxprio - 1) and mlock all qemu > and guest memory. > > Of course, we need more improvements to keep latency low in qemu > virtualized environment and this is a first step. OTOH, we can meet > the requirement of our first migration project with this patch. > > [ Note ] > This version doesn't support vhost, vpnc and linux-aio. > > These are some basic performance results: > > Host : 4 core, 4GB > Guest: 1 core, 512MB > > Benchmark: qpid-latency-test > http://qpid.apache.org/ > https://access.redhat.com/knowledge/docs/en-US/Red_Hat_Enterprise_MRG/2/html/Messaging_Installation_and_Configuration_Guide/qpid_latency_test.html > > Command: > - qemu > $ qemu -smp 1 -m 512 -enable-kvm -netdev tap,id=hostnet1 -device > virtio-net-pci,netdev=hostnet1 -drive file=vm.img,if=virtio > (-realtime maxprio=99,policy=fifo) > > - benchmark > $ chrt -f 99 qpid-latency-test --tcp-nodelay --rate 10000 -b > <server> > > Results: worst latency (msec) from 100 run > - no load > 1. normal qemu : 17.468400 > 2. chrt qemu(*) : 10.019900 > 3. realtime qemu: 8.048370 > > - load (iperf, server:vm, client:other physical sercer) > 4. normal qemu : 26.711100 > 5. chrt qemu : 8.485140 > 6. realtime qemu: 10.176700 > > (*) $ chrt -f -p 99 <event_thread_tid> > $ chrt -f -p 98 <vcpu_thread_tid> > > Any comments are welcome. > > Regards, > Satoru > > Signed-off-by: Satoru Moriya <satoru.mor...@hds.com> > --- > cpus.c | 17 +++++++++++++++++ > include/qemu/thread.h | 4 ++++ > include/sysemu/os-posix.h | 1 + > include/sysemu/os-win32.h | 1 + > os-posix.c | 48 > +++++++++++++++++++++++++++++++++++++++++++++++ > qemu-config.c | 16 ++++++++++++++++ > qemu-options.hx | 9 +++++++++ > qemu-thread-posix.c | 27 ++++++++++++++++++++++++++ > qemu-thread-win32.c | 13 +++++++++++++ > vl.c | 33 ++++++++++++++++++++++++++++++++ > 10 files changed, 169 insertions(+) > > diff --git a/cpus.c b/cpus.c > index 4a7782a..a049970 100644 > --- a/cpus.c > +++ b/cpus.c > @@ -734,6 +734,9 @@ static void *qemu_kvm_cpu_thread_fn(void *arg) > CPUArchState *env = arg; > CPUState *cpu = ENV_GET_CPU(env); > int r; > + int rt_policy, rt_priority; > + struct sched_param sp; > + > > qemu_mutex_lock(&qemu_global_mutex); > qemu_thread_get_self(cpu->thread); > @@ -746,6 +749,20 @@ static void *qemu_kvm_cpu_thread_fn(void *arg) > exit(1); > } > > + if (qemu_realtime_is_enabled()) { > + qemu_realtime_get_parameters(&rt_policy, &rt_priority); > + /* > + * vcpu threads' priority must be set to event thread priority -1 > + * to avoid starvation. > + */ > + sp.sched_priority = rt_priority - 1; > + r = sched_setscheduler(0, rt_policy, &sp); > + if (r < 0) { > + perror("Setting realtime policy failed"); > + exit(1); > + } > + } > + > qemu_kvm_init_cpu_signals(env); > > /* signal CPU creation */ > diff --git a/include/qemu/thread.h b/include/qemu/thread.h index > c02404b..3d8b3d2 100644 > --- a/include/qemu/thread.h > +++ b/include/qemu/thread.h > @@ -53,4 +53,8 @@ void qemu_thread_get_self(QemuThread *thread); bool > qemu_thread_is_self(QemuThread *thread); void qemu_thread_exit(void > *retval); > > +void qemu_init_realtime(int, int); > +bool qemu_realtime_is_enabled(void); > +void qemu_realtime_get_parameters(int *, int *); > + > #endif > diff --git a/include/sysemu/os-posix.h b/include/sysemu/os-posix.h > index 7f198e4..e5995b0 100644 > --- a/include/sysemu/os-posix.h > +++ b/include/sysemu/os-posix.h > @@ -31,6 +31,7 @@ void os_set_proc_name(const char *s); void > os_setup_signal_handling(void); void os_daemonize(void); void > os_setup_post(void); > +void os_prioritize(const char *, int); > > typedef struct timeval qemu_timeval; > #define qemu_gettimeofday(tp) gettimeofday(tp, NULL) diff --git > a/include/sysemu/os-win32.h b/include/sysemu/os-win32.h index > d0e9234..946b566 100644 > --- a/include/sysemu/os-win32.h > +++ b/include/sysemu/os-win32.h > @@ -78,6 +78,7 @@ static inline void os_daemonize(void) {} static > inline void os_setup_post(void) {} void os_set_line_buffering(void); > static inline void os_set_proc_name(const char *dummy) {} > +static inline void os_prioritize(const char *pol, int prio) {} > > #if !defined(EPROTONOSUPPORT) > # define EPROTONOSUPPORT EINVAL > diff --git a/os-posix.c b/os-posix.c > index 5c64518..8fe0fa2 100644 > --- a/os-posix.c > +++ b/os-posix.c > @@ -33,12 +33,14 @@ > #include <pwd.h> > #include <grp.h> > #include <libgen.h> > +#include <sched.h> > > /* Needed early for CONFIG_BSD etc. */ #include "config-host.h" > #include "sysemu/sysemu.h" > #include "net/slirp.h" > #include "qemu-options.h" > +#include "qemu-thread.h" > > #ifdef CONFIG_LINUX > #include <sys/prctl.h> > @@ -363,3 +365,49 @@ bool is_daemonized(void) { > return daemonize; > } > + > +void os_prioritize(const char *rt_sched_policy, int > +max_sched_priority) { > + int rt_pol, sys_min_prio, sys_max_prio; > + > + if (rt_sched_policy) { > + if (!strcmp(rt_sched_policy, "rr")) { > + rt_pol = SCHED_RR; > + } else if (!strcmp(rt_sched_policy, "fifo")) { > + rt_pol = SCHED_FIFO; > + } else { > + fprintf(stderr, "qemu: invalid option value '%s'\n", > + rt_sched_policy); > + exit(1); > + } > + } else { > + rt_pol = SCHED_RR; > + } > + > + sys_min_prio = sched_get_priority_min(rt_pol); > + sys_max_prio = sched_get_priority_max(rt_pol); > + > + if (max_sched_priority < sys_min_prio + 1) { > + /* > + * We set event threads' priority to max_sched_priorty and > + * vcpu threads' to max_sched_priority - 1 in order to avoid > + * starvation. So, it must be > sys_min_prio + 1. > + */ > + fprintf(stderr, "qemu: invalid option maxprio=%d. It must be >= > %d\n", > + max_sched_priority, sys_min_prio + 1); > + exit(1); > + } > + > + if (sys_max_prio < max_sched_priority) { > + fprintf(stderr, "qemu: invalid option maxprio=%d. It must be <= > %d\n", > + max_sched_priority, sys_max_prio); > + exit(1); > + } > + > + qemu_init_realtime(rt_pol, max_sched_priority); > + > + if (mlockall(MCL_CURRENT | MCL_FUTURE)) { > + perror("mlockall"); > + exit(1); > + } > +} > diff --git a/qemu-config.c b/qemu-config.c index 2188c3e..b945d07 > 100644 > --- a/qemu-config.c > +++ b/qemu-config.c > @@ -647,6 +647,21 @@ static QemuOptsList qemu_object_opts = { > }, > }; > > +static QemuOptsList qemu_realtime_opts = { > + .name = "realtime", > + .head = QTAILQ_HEAD_INITIALIZER(qemu_realtime_opts.head), > + .desc = { > + { > + .name = "maxprio", > + .type = QEMU_OPT_NUMBER, > + }, { > + .name = "policy", > + .type = QEMU_OPT_STRING, > + }, > + { /* end of list */ } > + }, > +}; > + > static QemuOptsList *vm_config_groups[32] = { > &qemu_drive_opts, > &qemu_chardev_opts, > @@ -664,6 +679,7 @@ static QemuOptsList *vm_config_groups[32] = { > &qemu_sandbox_opts, > &qemu_add_fd_opts, > &qemu_object_opts, > + &qemu_realtime_opts, > NULL, > }; > > diff --git a/qemu-options.hx b/qemu-options.hx index 9df0cde..968a20a > 100644 > --- a/qemu-options.hx > +++ b/qemu-options.hx > @@ -2447,6 +2447,15 @@ STEXI > Do not start CPU at startup (you must type 'c' in the monitor). > ETEXI > > +DEF("realtime", HAS_ARG, QEMU_OPTION_realtime, > + "-realtime maxprio=prio[,policy=pol]\n", > + QEMU_ARCH_ALL) > +STEXI > +@item -realtime maxprio=@var{prio}[,policy=@var{pold}] > +@findex -realtime > +run qemu as realtime process with priority @var{prio} and policy @var{pol}. > +ETEXI > + > DEF("gdb", HAS_ARG, QEMU_OPTION_gdb, \ > "-gdb dev wait for gdb connection on 'dev'\n", QEMU_ARCH_ALL) > STEXI > diff --git a/qemu-thread-posix.c b/qemu-thread-posix.c index > 7be292e..10a97cc 100644 > --- a/qemu-thread-posix.c > +++ b/qemu-thread-posix.c > @@ -22,6 +22,15 @@ > #include <sys/time.h> > #include "qemu/thread.h" > > +struct QemuRealtimeInfo { > + bool is_realtime; > + int policy; > + int max_priority; > +}; > +typedef struct QemuRealtimeInfo QemuRealtimeInfo; > + > +static QemuRealtimeInfo rt_info; > + > static void error_exit(int err, const char *msg) { > fprintf(stderr, "qemu: %s: %s\n", msg, strerror(err)); @@ -324,3 > +333,21 @@ void *qemu_thread_join(QemuThread *thread) > } > return ret; > } > + > +void qemu_init_realtime(int rt_sched_policy, int max_sched_priority) > +{ > + rt_info.is_realtime = true; > + rt_info.policy = rt_sched_policy; > + rt_info.max_priority = max_sched_priority; } > + > +bool qemu_realtime_is_enabled(void) > +{ > + return rt_info.is_realtime; > +} > + > +void qemu_realtime_get_parameters(int *policy, int *max_priority) { > + *policy = rt_info.policy; > + *max_priority = rt_info.max_priority; } > diff --git a/qemu-thread-win32.c b/qemu-thread-win32.c index > 8037b39..3beebcf 100644 > --- a/qemu-thread-win32.c > +++ b/qemu-thread-win32.c > @@ -369,3 +369,16 @@ bool qemu_thread_is_self(QemuThread *thread) { > return GetCurrentThreadId() == thread->tid; } > + > +void qemu_init_realtime(int rt_sched_policy, int max_sched_priority) > +{ } > + > +bool qemu_realtime_is_enabled(void) > +{ > + return false; > +} > + > +void qemu_realtime_get_parameters(int *policy, int *max_priority) { } > diff --git a/vl.c b/vl.c > index e6a8d89..c310587 100644 > --- a/vl.c > +++ b/vl.c > @@ -29,6 +29,7 @@ > #include <sys/time.h> > #include <zlib.h> > #include "qemu/bitmap.h" > +#include "qemu-thread.h" > > /* Needed early for CONFIG_BSD etc. */ #include "config-host.h" > @@ -1148,6 +1149,17 @@ static void smp_parse(const char *optarg) > max_cpus = smp_cpus; > } > > +static void configure_realtime(QemuOpts *opts) { > + const char *pol; > + int prio; > + > + pol = qemu_opt_get(opts, "policy"); > + prio = qemu_opt_get_number(opts, "maxprio", 1); > + > + os_prioritize(pol, prio); > +} > + > /***********************************************************/ > /* USB devices */ > > @@ -1754,9 +1766,22 @@ static void main_loop(void) { > bool nonblocking; > int last_io = 0; > + int rt_policy, rt_priority; > + struct sched_param sp; > #ifdef CONFIG_PROFILER > int64_t ti; > #endif > + > + if (qemu_realtime_is_enabled()) { > + qemu_realtime_get_parameters(&rt_policy, &rt_priority); > + > + sp.sched_priority = rt_priority;; > + if (sched_setscheduler(0, rt_policy, &sp) < 0) { > + perror("Setting realtime policy failed"); > + exit(1); > + } > + } > + > do { > nonblocking = !kvm_enabled() && last_io > 0; #ifdef > CONFIG_PROFILER @@ -2758,6 +2783,14 @@ int main(int argc, char **argv, > char **envp) > } > numa_add(optarg); > break; > + case QEMU_OPTION_realtime: > + opts = qemu_opts_parse(qemu_find_opts("realtime"), optarg, > 0); > + if (!opts) { > + fprintf(stderr, "parse error: %s\n", optarg); > + exit(1); > + } > + configure_realtime(opts); > + break; > case QEMU_OPTION_display: > display_type = select_display(optarg); > break; >
