SLUB: use have_arch_cmpxchg()
* Christoph Lameter ([EMAIL PROTECTED]) wrote:
> If we really want to do this then the implementation of all of these
> components need to result in competitive performance on all platforms.
>
Patch 2 of 2
Use have_arch_cmpxchg() to select between irq disable and cmpxchg_local.
Pro:
- Should provide the same performances on architectures that does not provide
fast local_cmpxchg.
- Improved performances on architectures that provide fast local_cmpxchg.
Cons:
- Does not help code readability, i.e.:
if (have_arch_cmpxchg())
preempt_disable();
else
local_irq_save(flags);
It applies on top of the cmpxchg_local() slub patch.
It depends on the Add have_arch_cmpxchg() patch.
Signed-off-by: Mathieu Desnoyers <[EMAIL PROTECTED]>
---
mm/slub.c | 65 +++++++++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 46 insertions(+), 19 deletions(-)
Index: slab/mm/slub.c
===================================================================
--- slab.orig/mm/slub.c 2007-08-22 10:28:22.000000000 -0400
+++ slab/mm/slub.c 2007-08-22 10:51:53.000000000 -0400
@@ -1450,7 +1450,8 @@ static void *__slab_alloc(struct kmem_ca
void **freelist = NULL;
unsigned long flags;
- local_irq_save(flags);
+ if (have_arch_cmpxchg())
+ local_irq_save(flags);
c = get_cpu_slab(s, smp_processor_id());
if (!c->page)
/* Slab was flushed */
@@ -1482,7 +1483,8 @@ out_object:
c->freelist = object[c->offset];
out:
slab_unlock(c->page);
- local_irq_restore(flags);
+ if (have_arch_cmpxchg())
+ local_irq_restore(flags);
if (unlikely((gfpflags & __GFP_ZERO)))
memset(object, 0, c->objsize);
return object;
@@ -1525,7 +1527,8 @@ new_slab:
c->page = new;
goto load_freelist;
}
- local_irq_restore(flags);
+ if (have_arch_cmpxchg())
+ local_irq_restore(flags);
return NULL;
debug:
object = c->page->freelist;
@@ -1553,8 +1556,12 @@ static void __always_inline *slab_alloc(
{
void **object;
struct kmem_cache_cpu *c;
+ unsigned long flags;
- preempt_disable();
+ if (have_arch_cmpxchg())
+ preempt_disable();
+ else
+ local_irq_save(flags);
c = get_cpu_slab(s, raw_smp_processor_id());
if (unlikely(!node_match(c, node)))
goto slow;
@@ -1564,15 +1571,21 @@ redo:
if (unlikely(!object))
goto slow;
- if (unlikely(cmpxchg_local(&c->freelist, object,
- object[c->offset]) != object))
- goto redo;
- preempt_enable();
+ if (have_arch_cmpxchg()) {
+ if (unlikely(cmpxchg_local(&c->freelist, object,
+ object[c->offset]) != object))
+ goto redo;
+ preempt_enable();
+ } else {
+ c->freelist = object[c->offset];
+ local_irq_restore(flags);
+ }
if (unlikely((gfpflags & __GFP_ZERO)))
memset(object, 0, c->objsize);
return object;
slow:
- preempt_enable();
+ if (have_arch_cmpxchg())
+ preempt_enable();
return __slab_alloc(s, gfpflags, node, addr);
}
@@ -1605,7 +1618,8 @@ static void __slab_free(struct kmem_cach
void **object = (void *)x;
unsigned long flags;
- local_irq_save(flags);
+ if (have_arch_cmpxchg())
+ local_irq_save(flags);
slab_lock(page);
if (unlikely(SlabDebug(page)))
@@ -1631,7 +1645,8 @@ checks_ok:
out_unlock:
slab_unlock(page);
- local_irq_restore(flags);
+ if (have_arch_cmpxchg())
+ local_irq_restore(flags);
return;
slab_empty:
@@ -1642,7 +1657,8 @@ slab_empty:
remove_partial(s, page);
slab_unlock(page);
- local_irq_restore(flags);
+ if (have_arch_cmpxchg())
+ local_irq_restore(flags);
discard_slab(s, page);
return;
@@ -1669,27 +1685,38 @@ static void __always_inline slab_free(st
void **object = (void *)x;
void **freelist;
struct kmem_cache_cpu *c;
+ unsigned long flags;
- preempt_disable();
+ if (have_arch_cmpxchg())
+ preempt_disable();
+ else
+ local_irq_save(flags);
c = get_cpu_slab(s, raw_smp_processor_id());
if (unlikely(c->node < 0))
goto slow;
redo:
freelist = c->freelist;
- barrier(); /* Read freelist before page, wrt local interrupts */
+ if (have_arch_cmpxchg())
+ barrier(); /* Read freelist before page, wrt local interrupts */
if (unlikely(page != c->page))
goto slow;
object[c->offset] = freelist;
- if (unlikely(cmpxchg_local(&c->freelist,
- freelist, object) != freelist))
- goto redo;
- preempt_enable();
+ if (have_arch_cmpxchg()) {
+ if (unlikely(cmpxchg_local(&c->freelist,
+ freelist, object) != freelist))
+ goto redo;
+ preempt_enable();
+ } else {
+ c->freelist = object;
+ local_irq_restore(flags);
+ }
return;
slow:
- preempt_enable();
+ if (have_arch_cmpxchg())
+ preempt_enable();
__slab_free(s, page, x, addr, c->offset);
}
--
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
