On 13-04-01 11:32 AM, Christoph Lameter wrote:
> On Thu, 28 Mar 2013, Paul Gortmaker wrote:
> 
>>> Index: linux/init/Kconfig
>>> ===================================================================
>>> --- linux.orig/init/Kconfig     2013-03-28 12:14:26.958358688 -0500
>>> +++ linux/init/Kconfig  2013-03-28 12:19:46.275866639 -0500
>>> @@ -1514,6 +1514,14 @@ config SLOB
>>>
>>>  endchoice
>>>
>>> +config SLUB_CPU_PARTIAL
>>> +       depends on SLUB
>>> +       bool "SLUB per cpu partial cache"
>>> +       help
>>> +         Per cpu partial caches accellerate freeing of objects at the
>>> +         price of more indeterminism in the latency of the free.
>>> +         Typically one would choose no for a realtime system.
>>
>> Is "batch" a better description than "accelerate" ?  Something like
> 
> Its not a batching but a cache that is going to be mainly used for new
> allocations on the same processor.

OK.  In that case, a minor nit - since it is user facing text, we should
probably drop one of the "l" for "accelerate".

> 
>>      Per cpu partial caches allows batch freeing of objects to maximize
>>      throughput.  However, this can increase the length of time spent
>>      holding key locks, which can increase latency spikes with respect
>>      to responsiveness.  Select yes unless you are tuning for a realtime
>>      oriented system.
>>
>> Also, I believe this will cause a behaviour change for people who
>> just run "make oldconfig" -- since there is no default line.  Meaning
>> that it used to be unconditionally on, but now I think it will be off
>> by default, if people just mindlessly hold down Enter key.
> 
> Ok.
>>
>> For RT, we'll want default N if RT_FULL (RT_BASE?) but for mainline,
>> I expect you'll want default Y in order to be consistent with previous
>> behaviour?
> 
> I was not sure exactly how to handle that one yet for realtime. So I need
> two different patches?

I don't think you need to worry about realtime -- meaning that I would guess
once the patch exists in mainline, Steve will probably cherry pick it onto
3.6.11.x-stable, and then he'd likely add a one-line follow on RT-specific
patch to make it set to off/disabled for RT.  Similar for 3.4.x etc.

> 
>> I've not built/booted yet, but I'll follow up if I see anything else in doing
>> that.
> 
> Here is an updated patch. I will also send an updated fixup patch.

I'll give these some local testing today.

Thanks,
Paul.
--

> 
> 
> Subject: slub: Make cpu partial slab support configurable V2
> 
> cpu partial support can introduce level of indeterminism that is not wanted
> in certain context (like a realtime kernel). Make it configurable.
> 
> Signed-off-by: Christoph Lameter <c...@linux.com>
> 
> Index: linux/include/linux/slub_def.h
> ===================================================================
> --- linux.orig/include/linux/slub_def.h       2013-04-01 10:27:05.908964674 
> -0500
> +++ linux/include/linux/slub_def.h    2013-04-01 10:27:19.905178531 -0500
> @@ -47,7 +47,9 @@ struct kmem_cache_cpu {
>       void **freelist;        /* Pointer to next available object */
>       unsigned long tid;      /* Globally unique transaction id */
>       struct page *page;      /* The slab from which we are allocating */
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>       struct page *partial;   /* Partially allocated frozen slabs */
> +#endif
>  #ifdef CONFIG_SLUB_STATS
>       unsigned stat[NR_SLUB_STAT_ITEMS];
>  #endif
> @@ -84,7 +86,9 @@ struct kmem_cache {
>       int size;               /* The size of an object including meta data */
>       int object_size;        /* The size of an object without meta data */
>       int offset;             /* Free pointer offset. */
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>       int cpu_partial;        /* Number of per cpu partial objects to keep 
> around */
> +#endif
>       struct kmem_cache_order_objects oo;
> 
>       /* Allocation and freeing of slabs */
> Index: linux/mm/slub.c
> ===================================================================
> --- linux.orig/mm/slub.c      2013-04-01 10:27:05.908964674 -0500
> +++ linux/mm/slub.c   2013-04-01 10:27:19.905178531 -0500
> @@ -1531,7 +1531,9 @@ static inline void *acquire_slab(struct
>       return freelist;
>  }
> 
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  static int put_cpu_partial(struct kmem_cache *s, struct page *page, int 
> drain);
> +#endif
>  static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
> 
>  /*
> @@ -1570,10 +1572,20 @@ static void *get_partial_node(struct kme
>                       object = t;
>                       available =  page->objects - (unsigned 
> long)page->lru.next;
>               } else {
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>                       available = put_cpu_partial(s, page, 0);
>                       stat(s, CPU_PARTIAL_NODE);
> +#else
> +                     BUG();
> +#endif
>               }
> -             if (kmem_cache_debug(s) || available > s->cpu_partial / 2)
> +             if (kmem_cache_debug(s) ||
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
> +                     available > s->cpu_partial / 2
> +#else
> +                     available > 0
> +#endif
> +                     )
>                       break;
> 
>       }
> @@ -1874,6 +1886,7 @@ redo:
>       }
>  }
> 
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  /*
>   * Unfreeze all the cpu partial slabs.
>   *
> @@ -1989,6 +2002,7 @@ static int put_cpu_partial(struct kmem_c
>       } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) != 
> oldpage);
>       return pobjects;
>  }
> +#endif
> 
>  static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
>  {
> @@ -2013,7 +2027,9 @@ static inline void __flush_cpu_slab(stru
>               if (c->page)
>                       flush_slab(s, c);
> 
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>               unfreeze_partials(s, c);
> +#endif
>       }
>  }
> 
> @@ -2029,7 +2045,11 @@ static bool has_cpu_slab(int cpu, void *
>       struct kmem_cache *s = info;
>       struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
> 
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>       return c->page || c->partial;
> +#else
> +     return c->page;
> +#endif
>  }
> 
>  static void flush_all(struct kmem_cache *s)
> @@ -2225,7 +2245,10 @@ static void *__slab_alloc(struct kmem_ca
>       page = c->page;
>       if (!page)
>               goto new_slab;
> +
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  redo:
> +#endif
> 
>       if (unlikely(!node_match(page, node))) {
>               stat(s, ALLOC_NODE_MISMATCH);
> @@ -2278,6 +2301,7 @@ load_freelist:
> 
>  new_slab:
> 
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>       if (c->partial) {
>               page = c->page = c->partial;
>               c->partial = page->next;
> @@ -2285,6 +2309,7 @@ new_slab:
>               c->freelist = NULL;
>               goto redo;
>       }
> +#endif
> 
>       freelist = new_slab_objects(s, gfpflags, node, &c);
> 
> @@ -2491,6 +2516,7 @@ static void __slab_free(struct kmem_cach
>               new.inuse--;
>               if ((!new.inuse || !prior) && !was_frozen) {
> 
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>                       if (!kmem_cache_debug(s) && !prior)
> 
>                               /*
> @@ -2499,7 +2525,9 @@ static void __slab_free(struct kmem_cach
>                                */
>                               new.frozen = 1;
> 
> -                     else { /* Needs to be taken off a list */
> +                     else
> +#endif
> +                             { /* Needs to be taken off a list */
> 
>                               n = get_node(s, page_to_nid(page));
>                               /*
> @@ -2521,6 +2549,7 @@ static void __slab_free(struct kmem_cach
>               "__slab_free"));
> 
>       if (likely(!n)) {
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
> 
>               /*
>                * If we just froze the page then put it onto the
> @@ -2530,6 +2559,7 @@ static void __slab_free(struct kmem_cach
>                       put_cpu_partial(s, page, 1);
>                       stat(s, CPU_PARTIAL_FREE);
>               }
> +#endif
>               /*
>                * The list lock was not taken therefore no list
>                * activity can be necessary.
> @@ -3036,7 +3066,7 @@ static int kmem_cache_open(struct kmem_c
>        * list to avoid pounding the page allocator excessively.
>        */
>       set_min_partial(s, ilog2(s->size) / 2);
> -
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>       /*
>        * cpu_partial determined the maximum number of objects kept in the
>        * per cpu partial lists of a processor.
> @@ -3064,6 +3094,7 @@ static int kmem_cache_open(struct kmem_c
>               s->cpu_partial = 13;
>       else
>               s->cpu_partial = 30;
> +#endif
> 
>  #ifdef CONFIG_NUMA
>       s->remote_node_defrag_ratio = 1000;
> @@ -4424,13 +4455,14 @@ static ssize_t show_slab_objects(struct
>                       total += x;
>                       nodes[node] += x;
> 
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>                       page = ACCESS_ONCE(c->partial);
>                       if (page) {
>                               x = page->pobjects;
>                               total += x;
>                               nodes[node] += x;
>                       }
> -
> +#endif
>                       per_cpu[node]++;
>               }
>       }
> @@ -4583,6 +4615,7 @@ static ssize_t min_partial_store(struct
>  }
>  SLAB_ATTR(min_partial);
> 
> +#ifdef CONFIG_CPU_PARTIAL
>  static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
>  {
>       return sprintf(buf, "%u\n", s->cpu_partial);
> @@ -4605,6 +4638,7 @@ static ssize_t cpu_partial_store(struct
>       return length;
>  }
>  SLAB_ATTR(cpu_partial);
> +#endif
> 
>  static ssize_t ctor_show(struct kmem_cache *s, char *buf)
>  {
> @@ -4644,6 +4678,7 @@ static ssize_t objects_partial_show(stru
>  }
>  SLAB_ATTR_RO(objects_partial);
> 
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>  static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
>  {
>       int objects = 0;
> @@ -4674,6 +4709,7 @@ static ssize_t slabs_cpu_partial_show(st
>       return len + sprintf(buf + len, "\n");
>  }
>  SLAB_ATTR_RO(slabs_cpu_partial);
> +#endif
> 
>  static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
>  {
> @@ -4997,11 +5033,13 @@ STAT_ATTR(DEACTIVATE_BYPASS, deactivate_
>  STAT_ATTR(ORDER_FALLBACK, order_fallback);
>  STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
>  STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
> +#ifdef CONFIG_CPU_PARTIAL
>  STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
>  STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
>  STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
>  STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
>  #endif
> +#endif
> 
>  static struct attribute *slab_attrs[] = {
>       &slab_size_attr.attr,
> @@ -5009,7 +5047,9 @@ static struct attribute *slab_attrs[] =
>       &objs_per_slab_attr.attr,
>       &order_attr.attr,
>       &min_partial_attr.attr,
> +#ifdef CONFIG_CPU_PARTIAL
>       &cpu_partial_attr.attr,
> +#endif
>       &objects_attr.attr,
>       &objects_partial_attr.attr,
>       &partial_attr.attr,
> @@ -5022,7 +5062,9 @@ static struct attribute *slab_attrs[] =
>       &destroy_by_rcu_attr.attr,
>       &shrink_attr.attr,
>       &reserved_attr.attr,
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>       &slabs_cpu_partial_attr.attr,
> +#endif
>  #ifdef CONFIG_SLUB_DEBUG
>       &total_objects_attr.attr,
>       &slabs_attr.attr,
> @@ -5064,11 +5106,13 @@ static struct attribute *slab_attrs[] =
>       &order_fallback_attr.attr,
>       &cmpxchg_double_fail_attr.attr,
>       &cmpxchg_double_cpu_fail_attr.attr,
> +#ifdef CONFIG_SLUB_CPU_PARTIAL
>       &cpu_partial_alloc_attr.attr,
>       &cpu_partial_free_attr.attr,
>       &cpu_partial_node_attr.attr,
>       &cpu_partial_drain_attr.attr,
>  #endif
> +#endif
>  #ifdef CONFIG_FAILSLAB
>       &failslab_attr.attr,
>  #endif
> Index: linux/init/Kconfig
> ===================================================================
> --- linux.orig/init/Kconfig   2013-04-01 10:27:05.908964674 -0500
> +++ linux/init/Kconfig        2013-04-01 10:31:46.497863625 -0500
> @@ -1514,6 +1514,17 @@ config SLOB
> 
>  endchoice
> 
> +config SLUB_CPU_PARTIAL
> +     default y
> +     depends on SLUB
> +     bool "SLUB per cpu partial cache"
> +     help
> +       Per cpu partial caches accellerate objects allocation and freeing
> +       that is local to a processor at the price of more indeterminism
> +       in the latency of the free. On overflow these caches will be cleared
> +       which requires the taking of locks that may cause latency spikes.
> +       Typically one would choose no for a realtime system.
> +
>  config MMAP_ALLOW_UNINITIALIZED
>       bool "Allow mmapped anonymous memory to be uninitialized"
>       depends on EXPERT && !MMU
> 
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/

Reply via email to