On Thu, Jan 22, 2015 at 05:04:12PM +0000, Andrew Cooper wrote:
> On 22/01/15 16:52, Konrad Rzeszutek Wilk wrote:
> > Contemporary servers sport now 480 CPUs or such. We should crank
> > up the default amount of CPUs to a higher level to take advantage
> > of this without having the distro to use 'max_phys_cpus' override.
> >
> > Signed-off-by: Konrad Rzeszutek Wilk <konrad.w...@oracle.com>
>
> /me would really like to try something that big out, but I have not had
> the opportunity yet to hit the 256 limit.
>
> I wonder which variables grow as a result of this change. We might want
> to see about making more things dynamically allocated after reading the
> apci tables, if we can.
I am not sure if that is possible as there is a lot of DEFINE_PER_CPU
which are cannot grow.
The structures that grow are:
struct cpumask
struct kernel_param
struct rangeset
struct csched2_runqueue_data
struct csched2_private
struct rt_vcpu
struct stopmachine_data
struct free_ptr
struct rcu_data
struct physid_mask
struct acpi_table_header
struct calibration_rendezvous
struct bug_frame
(for fun see attached diff of pahole between 256 and 512 CPUs)
>
> ~Andrew
>
> > ---
> > xen/include/asm-x86/config.h | 2 +-
> > 1 file changed, 1 insertion(+), 1 deletion(-)
> >
> > diff --git a/xen/include/asm-x86/config.h b/xen/include/asm-x86/config.h
> > index 2fbd68d..d450696 100644
> > --- a/xen/include/asm-x86/config.h
> > +++ b/xen/include/asm-x86/config.h
> > @@ -64,7 +64,7 @@
> > #ifdef MAX_PHYS_CPUS
> > #define NR_CPUS MAX_PHYS_CPUS
> > #else
> > -#define NR_CPUS 256
> > +#define NR_CPUS 512
> > #endif
> >
> > /* Linkage for x86 */
>
--- 256 2015-01-22 14:01:55.200283080 -0500
+++ 512 2015-01-22 13:59:08.588811566 -0500
@@ -1,8 +1,8 @@
struct cpumask {
- long unsigned int bits[4]; /* 0 32 */
+ long unsigned int bits[8]; /* 0 64 */
+ /* --- cacheline 1 boundary (64 bytes) --- */
- /* size: 32, cachelines: 1, members: 1 */
- /* last cacheline: 32 bytes */
+ /* size: 64, cachelines: 1, members: 1 */
};
struct kernel_param {
const char * name; /* 0 8 */
@@ -4853,10 +4853,11 @@
long unsigned int start; /* 8 8 */
long unsigned int per_cpu_sz; /* 16 8 */
long unsigned int rem; /* 24 8 */
- cpumask_t cpus; /* 32 32 */
- /* --- cacheline 1 boundary (64 bytes) --- */
+ cpumask_t cpus; /* 32 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 32 bytes ago --- */
- /* size: 64, cachelines: 1, members: 5 */
+ /* size: 96, cachelines: 2, members: 5 */
+ /* last cacheline: 32 bytes */
};
struct rangeset {
struct list_head rangeset_list; /* 0 16 */
@@ -4968,46 +4969,49 @@
struct csched2_runqueue_data {
int id; /* 0 4 */
spinlock_t lock; /* 4 4 */
- cpumask_t active; /* 8 32 */
- struct list_head runq; /* 40 16 */
- struct list_head svc; /* 56 16 */
+ cpumask_t active; /* 8 64 */
/* --- cacheline 1 boundary (64 bytes) was 8 bytes ago --- */
- unsigned int max_weight; /* 72 4 */
+ struct list_head runq; /* 72 16 */
+ struct list_head svc; /* 88 16 */
+ unsigned int max_weight; /* 104 4 */
/* XXX 4 bytes hole, try to pack */
- cpumask_t idle; /* 80 32 */
- cpumask_t tickled; /* 112 32 */
- /* --- cacheline 2 boundary (128 bytes) was 16 bytes ago --- */
- int load; /* 144 4 */
+ cpumask_t idle; /* 112 64 */
+ /* --- cacheline 2 boundary (128 bytes) was 48 bytes ago --- */
+ cpumask_t tickled; /* 176 64 */
+ /* --- cacheline 3 boundary (192 bytes) was 48 bytes ago --- */
+ int load; /* 240 4 */
/* XXX 4 bytes hole, try to pack */
- s_time_t load_last_update; /* 152 8 */
- s_time_t avgload; /* 160 8 */
- s_time_t b_avgload; /* 168 8 */
+ s_time_t load_last_update; /* 248 8 */
+ /* --- cacheline 4 boundary (256 bytes) --- */
+ s_time_t avgload; /* 256 8 */
+ s_time_t b_avgload; /* 264 8 */
- /* size: 176, cachelines: 3, members: 12 */
- /* sum members: 168, holes: 2, sum holes: 8 */
- /* last cacheline: 48 bytes */
+ /* size: 272, cachelines: 5, members: 12 */
+ /* sum members: 264, holes: 2, sum holes: 8 */
+ /* last cacheline: 16 bytes */
};
struct csched2_private {
spinlock_t lock; /* 0 4 */
/* XXX 4 bytes hole, try to pack */
- cpumask_t initialized; /* 8 32 */
- struct list_head sdom; /* 40 16 */
- int runq_map[256]; /* 56 1024 */
- /* --- cacheline 16 boundary (1024 bytes) was 56 bytes ago --- */
- cpumask_t active_queues; /* 1080 32 */
- /* --- cacheline 17 boundary (1088 bytes) was 24 bytes ago --- */
- struct csched2_runqueue_data rqd[256]; /* 1112 45056 */
- /* --- cacheline 721 boundary (46144 bytes) was 24 bytes ago --- */
- int load_window_shift; /* 46168 4 */
+ cpumask_t initialized; /* 8 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 8 bytes ago --- */
+ struct list_head sdom; /* 72 16 */
+ int runq_map[512]; /* 88 2048 */
+ /* --- cacheline 33 boundary (2112 bytes) was 24 bytes ago --- */
+ cpumask_t active_queues; /* 2136 64 */
+ /* --- cacheline 34 boundary (2176 bytes) was 24 bytes ago --- */
+ struct csched2_runqueue_data rqd[512]; /* 2200 139264 */
+ /* --- cacheline 2210 boundary (141440 bytes) was 24 bytes ago --- */
+ int load_window_shift; /* 141464 4 */
- /* size: 46176, cachelines: 722, members: 7 */
- /* sum members: 46168, holes: 1, sum holes: 4 */
+ /* size: 141472, cachelines: 2211, members: 7 */
+ /* sum members: 141464, holes: 1, sum holes: 4 */
/* padding: 4 */
/* last cacheline: 32 bytes */
};
@@ -5149,12 +5153,12 @@
struct list_head sdom; /* 8 16 */
struct list_head runq; /* 24 16 */
struct list_head depletedq; /* 40 16 */
- cpumask_t tickled; /* 56 32 */
- /* --- cacheline 1 boundary (64 bytes) was 24 bytes ago --- */
+ cpumask_t tickled; /* 56 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 56 bytes ago --- */
- /* size: 88, cachelines: 2, members: 5 */
- /* sum members: 84, holes: 1, sum holes: 4 */
- /* last cacheline: 24 bytes */
+ /* size: 120, cachelines: 2, members: 5 */
+ /* sum members: 116, holes: 1, sum holes: 4 */
+ /* last cacheline: 56 bytes */
};
struct rt_vcpu {
struct list_head q_elem; /* 0 16 */
@@ -5232,11 +5236,12 @@
/* XXX 4 bytes hole, try to pack */
- cpumask_t selected; /* 24 32 */
+ cpumask_t selected; /* 24 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 24 bytes ago --- */
- /* size: 56, cachelines: 1, members: 4 */
- /* sum members: 52, holes: 1, sum holes: 4 */
- /* last cacheline: 56 bytes */
+ /* size: 88, cachelines: 2, members: 4 */
+ /* sum members: 84, holes: 1, sum holes: 4 */
+ /* last cacheline: 24 bytes */
};
struct stopmachine_data {
unsigned int nr_cpus; /* 0 4 */
@@ -5702,13 +5707,13 @@
struct vcpu * vcpu; /* 16 8 */
void * esp; /* 24 8 */
char * stack; /* 32 8 */
- cpumask_t saved_affinity; /* 40 32 */
- /* --- cacheline 1 boundary (64 bytes) was 8 bytes ago --- */
- unsigned int wakeup_cpu; /* 72 4 */
+ cpumask_t saved_affinity; /* 40 64 */
+ /* --- cacheline 1 boundary (64 bytes) was 40 bytes ago --- */
+ unsigned int wakeup_cpu; /* 104 4 */
- /* size: 80, cachelines: 2, members: 6 */
+ /* size: 112, cachelines: 2, members: 6 */
/* padding: 4 */
- /* last cacheline: 16 bytes */
+ /* last cacheline: 48 bytes */
};
struct free_ptr {
struct bhdr * prev; /* 0 8 */
@@ -5774,11 +5779,12 @@
/* XXX 4 bytes hole, try to pack */
- cpumask_t cpumask; /* 136 32 */
+ cpumask_t cpumask; /* 136 64 */
+ /* --- cacheline 3 boundary (192 bytes) was 8 bytes ago --- */
/* size: 256, cachelines: 4, members: 5 */
- /* sum members: 56, holes: 2, sum holes: 112 */
- /* padding: 88 */
+ /* sum members: 88, holes: 2, sum holes: 112 */
+ /* padding: 56 */
};
struct rcu_data {
long int quiescbatch; /* 0 8 */
@@ -7630,10 +7636,10 @@
/* last cacheline: 8 bytes */
};
struct physid_mask {
- long unsigned int mask[16]; /* 0 128 */
- /* --- cacheline 2 boundary (128 bytes) --- */
+ long unsigned int mask[32]; /* 0 256 */
+ /* --- cacheline 4 boundary (256 bytes) --- */
- /* size: 128, cachelines: 2, members: 1 */
+ /* size: 256, cachelines: 4, members: 1 */
};
struct acpi_table_header {
char signature[4]; /* 0 4 */
@@ -10672,17 +10678,18 @@
/* last cacheline: 24 bytes */
};
struct calibration_rendezvous {
- cpumask_t cpu_calibration_map; /* 0 32 */
- atomic_t semaphore; /* 32 4 */
+ cpumask_t cpu_calibration_map; /* 0 64 */
+ /* --- cacheline 1 boundary (64 bytes) --- */
+ atomic_t semaphore; /* 64 4 */
/* XXX 4 bytes hole, try to pack */
- s_time_t master_stime; /* 40 8 */
- u64 master_tsc_stamp; /* 48 8 */
+ s_time_t master_stime; /* 72 8 */
+ u64 master_tsc_stamp; /* 80 8 */
- /* size: 56, cachelines: 1, members: 4 */
- /* sum members: 52, holes: 1, sum holes: 4 */
- /* last cacheline: 56 bytes */
+ /* size: 88, cachelines: 2, members: 4 */
+ /* sum members: 84, holes: 1, sum holes: 4 */
+ /* last cacheline: 24 bytes */
};
struct bug_frame {
int loc_disp:24; /* 0: 8 4 */
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
