Author: jeff
Date: Mon Jun 17 03:43:47 2013
New Revision: 251826
URL: http://svnweb.freebsd.org/changeset/base/251826
Log:
- Add a new UMA API: uma_zcache_create(). This makes a zone without any
backing memory that is only a container for per-cpu caches of arbitrary
pointer items. These zones have no kegs.
- Convert the regular keg based allocator to use the new import/release
functions.
- Move some stats to be atomics since they would require excessive zone
locking/unlocking with the new import/release paradigm. Make
zone_free_item simpler now that callers can manage more stats.
- Check for these cache-only zones in the public APIs and debugging
code by checking zone_first_keg() against NULL.
Sponsored by: EMC / Isilong Storage Division
Modified:
head/sys/vm/uma.h
head/sys/vm/uma_core.c
head/sys/vm/uma_int.h
Modified: head/sys/vm/uma.h
==============================================================================
--- head/sys/vm/uma.h Mon Jun 17 03:32:27 2013 (r251825)
+++ head/sys/vm/uma.h Mon Jun 17 03:43:47 2013 (r251826)
@@ -124,6 +124,16 @@ typedef int (*uma_init)(void *mem, int s
typedef void (*uma_fini)(void *mem, int size);
/*
+ * Import new memory into a cache zone.
+ */
+typedef int (*uma_import)(void *arg, void **store, int count, int flags);
+
+/*
+ * Free memory from a cache zone.
+ */
+typedef void (*uma_release)(void *arg, void **store, int count);
+
+/*
* What's the difference between initializing and constructing?
*
* The item is initialized when it is cached, and this is the state that the
@@ -216,6 +226,19 @@ uma_zone_t uma_zsecond_create(char *name
int uma_zsecond_add(uma_zone_t zone, uma_zone_t master);
/*
+ * Create cache-only zones.
+ *
+ * This allows uma's per-cpu cache facilities to handle arbitrary
+ * pointers. Consumers must specify the import and release functions to
+ * fill and destroy caches. UMA does not allocate any memory for these
+ * zones. The 'arg' parameter is passed to import/release and is caller
+ * specific.
+ */
+uma_zone_t uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor,
+ uma_init zinit, uma_fini zfini, uma_import zimport,
+ uma_release zrelease, void *arg, int flags);
+
+/*
* Definitions for uma_zcreate flags
*
* These flags share space with UMA_ZFLAGs in uma_int.h. Be careful not to
Modified: head/sys/vm/uma_core.c
==============================================================================
--- head/sys/vm/uma_core.c Mon Jun 17 03:32:27 2013 (r251825)
+++ head/sys/vm/uma_core.c Mon Jun 17 03:43:47 2013 (r251826)
@@ -131,14 +131,14 @@ static int bucketdisable = 1;
static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs);
/* This mutex protects the keg list */
-static struct mtx uma_mtx;
+static struct mtx_padalign uma_mtx;
/* Linked list of boot time pages */
static LIST_HEAD(,uma_slab) uma_boot_pages =
LIST_HEAD_INITIALIZER(uma_boot_pages);
/* This mutex protects the boot time pages list */
-static struct mtx uma_boot_pages_mtx;
+static struct mtx_padalign uma_boot_pages_mtx;
/* Is the VM done starting up? */
static int booted = 0;
@@ -172,6 +172,9 @@ struct uma_zctor_args {
uma_dtor dtor;
uma_init uminit;
uma_fini fini;
+ uma_import import;
+ uma_release release;
+ void *arg;
uma_keg_t keg;
int align;
uint32_t flags;
@@ -216,9 +219,6 @@ static uint8_t bucket_size[BUCKET_ZONES]
*/
enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI };
-#define ZFREE_STATFAIL 0x00000001 /* Update zone failure
statistic. */
-#define ZFREE_STATFREE 0x00000002 /* Update zone free statistic.
*/
-
/* Prototypes.. */
static void *noobj_alloc(uma_zone_t, int, uint8_t *, int);
@@ -244,8 +244,7 @@ static void hash_free(struct uma_hash *h
static void uma_timeout(void *);
static void uma_startup3(void);
static void *zone_alloc_item(uma_zone_t, void *, int);
-static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip,
- int);
+static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip);
static void bucket_enable(void);
static void bucket_init(void);
static uma_bucket_t bucket_alloc(int, int);
@@ -254,11 +253,14 @@ static void bucket_zone_drain(void);
static int zone_alloc_bucket(uma_zone_t zone, int flags);
static uma_slab_t zone_fetch_slab(uma_zone_t zone, uma_keg_t last, int flags);
static uma_slab_t zone_fetch_slab_multi(uma_zone_t zone, uma_keg_t last, int
flags);
-static void *slab_alloc_item(uma_zone_t zone, uma_slab_t slab);
+static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab);
+static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item);
static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit,
uma_fini fini, int align, uint32_t flags);
static inline void zone_relock(uma_zone_t zone, uma_keg_t keg);
static inline void keg_relock(uma_keg_t keg, uma_zone_t zone);
+static int zone_import(uma_zone_t zone, void **bucket, int max, int flags);
+static void zone_release(uma_zone_t zone, void **bucket, int cnt);
void uma_print_zone(uma_zone_t);
void uma_print_stats(void);
@@ -363,8 +365,7 @@ bucket_free(uma_bucket_t bucket)
struct uma_bucket_zone *ubz;
ubz = bucket_zone_lookup(bucket->ub_entries);
- zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE,
- ZFREE_STATFREE);
+ zone_free_item(ubz->ubz_zone, bucket, NULL, SKIP_NONE);
}
static void
@@ -388,13 +389,6 @@ zone_log_warning(uma_zone_t zone)
printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning);
}
-static inline uma_keg_t
-zone_first_keg(uma_zone_t zone)
-{
-
- return (LIST_FIRST(&zone->uz_kegs)->kl_keg);
-}
-
static void
zone_foreach_keg(uma_zone_t zone, void (*kegfn)(uma_keg_t))
{
@@ -579,8 +573,7 @@ hash_free(struct uma_hash *hash)
if (hash->uh_slab_hash == NULL)
return;
if (hash->uh_hashsize == UMA_HASH_SIZE_INIT)
- zone_free_item(hashzone,
- hash->uh_slab_hash, NULL, SKIP_NONE, ZFREE_STATFREE);
+ zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE);
else
free(hash->uh_slab_hash, M_UMAHASH);
}
@@ -599,21 +592,16 @@ hash_free(struct uma_hash *hash)
static void
bucket_drain(uma_zone_t zone, uma_bucket_t bucket)
{
- void *item;
+ int i;
if (bucket == NULL)
return;
- while (bucket->ub_cnt > 0) {
- bucket->ub_cnt--;
- item = bucket->ub_bucket[bucket->ub_cnt];
-#ifdef INVARIANTS
- bucket->ub_bucket[bucket->ub_cnt] = NULL;
- KASSERT(item != NULL,
- ("bucket_drain: botched ptr, item is NULL"));
-#endif
- zone_free_item(zone, item, NULL, SKIP_DTOR, 0);
- }
+ if (zone->uz_fini)
+ for (i = 0; i < bucket->ub_cnt; i++)
+ zone->uz_fini(bucket->ub_bucket[i], zone->uz_size);
+ zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt);
+ bucket->ub_cnt = 0;
}
/*
@@ -767,8 +755,7 @@ finished:
obj);
}
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
- zone_free_item(keg->uk_slabzone, slab, NULL,
- SKIP_NONE, ZFREE_STATFREE);
+ zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
#ifdef UMA_DEBUG
printf("%s: Returning %d bytes.\n",
keg->uk_name, PAGE_SIZE * keg->uk_ppera);
@@ -842,7 +829,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t
slab = NULL;
#ifdef UMA_DEBUG
- printf("slab_zalloc: Allocating a new slab for %s\n", keg->uk_name);
+ printf("alloc_slab: Allocating a new slab for %s\n", keg->uk_name);
#endif
allocf = keg->uk_allocf;
KEG_UNLOCK(keg);
@@ -874,8 +861,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t
mem = allocf(zone, keg->uk_ppera * PAGE_SIZE, &flags, wait);
if (mem == NULL) {
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
- zone_free_item(keg->uk_slabzone, slab, NULL,
- SKIP_NONE, ZFREE_STATFREE);
+ zone_free_item(keg->uk_slabzone, slab, NULL, SKIP_NONE);
KEG_LOCK(keg);
return (NULL);
}
@@ -929,7 +915,7 @@ keg_alloc_slab(uma_keg_t keg, uma_zone_t
}
if (keg->uk_flags & UMA_ZONE_OFFPAGE)
zone_free_item(keg->uk_slabzone, slab,
- NULL, SKIP_NONE, ZFREE_STATFREE);
+ NULL, SKIP_NONE);
keg->uk_freef(mem, PAGE_SIZE * keg->uk_ppera,
flags);
KEG_LOCK(keg);
@@ -1483,6 +1469,24 @@ zone_ctor(void *mem, int size, void *uda
timevalclear(&zone->uz_ratecheck);
keg = arg->keg;
+ /*
+ * This is a pure cache zone, no kegs.
+ */
+ if (arg->import) {
+ zone->uz_import = arg->import;
+ zone->uz_release = arg->release;
+ zone->uz_arg = arg->arg;
+ zone->uz_count = BUCKET_MAX;
+ return (0);
+ }
+
+ /*
+ * Use the regular zone/keg/slab allocator.
+ */
+ zone->uz_import = (uma_import)zone_import;
+ zone->uz_release = (uma_release)zone_release;
+ zone->uz_arg = zone;
+
if (arg->flags & UMA_ZONE_SECONDARY) {
KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg"));
zone->uz_init = arg->uminit;
@@ -1519,6 +1523,7 @@ zone_ctor(void *mem, int size, void *uda
if (error)
return (error);
}
+
/*
* Link in the first keg.
*/
@@ -1616,12 +1621,11 @@ zone_dtor(void *arg, int size, void *uda
/*
* We only destroy kegs from non secondary zones.
*/
- if ((zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
+ if (keg != NULL && (zone->uz_flags & UMA_ZONE_SECONDARY) == 0) {
mtx_lock(&uma_mtx);
LIST_REMOVE(keg, uk_link);
mtx_unlock(&uma_mtx);
- zone_free_item(kegs, keg, NULL, SKIP_NONE,
- ZFREE_STATFREE);
+ zone_free_item(kegs, keg, NULL, SKIP_NONE);
}
}
@@ -1665,6 +1669,7 @@ uma_startup(void *bootmem, int boot_page
mtx_init(&uma_mtx, "UMA lock", NULL, MTX_DEF);
/* "manually" create the initial zone */
+ memset(&args, 0, sizeof(args));
args.name = "UMA Kegs";
args.size = sizeof(struct uma_keg);
args.ctor = keg_ctor;
@@ -1805,6 +1810,7 @@ uma_zcreate(const char *name, size_t siz
struct uma_zctor_args args;
/* This stuff is essential for the zone ctor */
+ memset(&args, 0, sizeof(args));
args.name = name;
args.size = size;
args.ctor = ctor;
@@ -1827,6 +1833,7 @@ uma_zsecond_create(char *name, uma_ctor
uma_keg_t keg;
keg = zone_first_keg(master);
+ memset(&args, 0, sizeof(args));
args.name = name;
args.size = keg->uk_size;
args.ctor = ctor;
@@ -1841,6 +1848,30 @@ uma_zsecond_create(char *name, uma_ctor
return (zone_alloc_item(zones, &args, M_WAITOK));
}
+/* See uma.h */
+uma_zone_t
+uma_zcache_create(char *name, uma_ctor ctor, uma_dtor dtor, uma_init zinit,
+ uma_fini zfini, uma_import zimport, uma_release zrelease,
+ void *arg, int flags)
+{
+ struct uma_zctor_args args;
+
+ memset(&args, 0, sizeof(args));
+ args.name = name;
+ args.size = 0;
+ args.ctor = ctor;
+ args.dtor = dtor;
+ args.uminit = zinit;
+ args.fini = zfini;
+ args.import = zimport;
+ args.release = zrelease;
+ args.arg = arg;
+ args.align = 0;
+ args.flags = flags;
+
+ return (zone_alloc_item(zones, &args, M_WAITOK));
+}
+
static void
zone_lock_pair(uma_zone_t a, uma_zone_t b)
{
@@ -1932,7 +1963,7 @@ void
uma_zdestroy(uma_zone_t zone)
{
- zone_free_item(zones, zone, NULL, SKIP_NONE, ZFREE_STATFREE);
+ zone_free_item(zones, zone, NULL, SKIP_NONE);
}
/* See uma.h */
@@ -2011,9 +2042,9 @@ zalloc_start:
if (zone->uz_ctor != NULL) {
if (zone->uz_ctor(item, zone->uz_size,
udata, flags) != 0) {
+ atomic_add_long(&zone->uz_fails, 1);
zone_free_item(zone, item, udata,
- SKIP_DTOR, ZFREE_STATFAIL |
- ZFREE_STATFREE);
+ SKIP_DTOR);
return (NULL);
}
}
@@ -2069,9 +2100,9 @@ zalloc_start:
}
/* Since we have locked the zone we may as well send back our stats */
- zone->uz_allocs += cache->uc_allocs;
+ atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
+ atomic_add_long(&zone->uz_frees, cache->uc_frees);
cache->uc_allocs = 0;
- zone->uz_frees += cache->uc_frees;
cache->uc_frees = 0;
/* Our old one is now a free bucket */
@@ -2319,13 +2350,12 @@ zone_fetch_slab_multi(uma_zone_t zone, u
}
static void *
-slab_alloc_item(uma_zone_t zone, uma_slab_t slab)
+slab_alloc_item(uma_keg_t keg, uma_slab_t slab)
{
- uma_keg_t keg;
void *item;
uint8_t freei;
- keg = slab->us_keg;
+ MPASS(keg == slab->us_keg);
mtx_assert(&keg->uk_lock, MA_OWNED);
freei = BIT_FFS(SLAB_SETSIZE, &slab->us_free) - 1;
@@ -2344,36 +2374,41 @@ slab_alloc_item(uma_zone_t zone, uma_sla
}
static int
-zone_alloc_bucket(uma_zone_t zone, int flags)
+zone_import(uma_zone_t zone, void **bucket, int max, int flags)
{
- uma_bucket_t bucket;
uma_slab_t slab;
uma_keg_t keg;
- int16_t saved;
- int max, origflags = flags;
-
- /*
- * Try this zone's free list first so we don't allocate extra buckets.
- */
- if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
- KASSERT(bucket->ub_cnt == 0,
- ("zone_alloc_bucket: Bucket on free list is not empty."));
- LIST_REMOVE(bucket, ub_link);
- } else {
- int bflags;
+ int i;
- bflags = (flags & ~M_ZERO);
- if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
- bflags |= M_NOVM;
+ ZONE_LOCK(zone);
+ /* Try to keep the buckets totally full */
+ slab = NULL;
+ keg = NULL;
+ for (i = 0; i < max; ) {
+ if ((slab = zone->uz_slab(zone, keg, flags)) == NULL)
+ break;
+ keg = slab->us_keg;
+ while (slab->us_freecount && i < max)
+ bucket[i++] = slab_alloc_item(keg, slab);
- ZONE_UNLOCK(zone);
- bucket = bucket_alloc(zone->uz_count, bflags);
- ZONE_LOCK(zone);
+ /* Don't block on the next fill */
+ flags &= ~M_WAITOK;
+ flags |= M_NOWAIT;
}
+ if (slab != NULL)
+ KEG_UNLOCK(keg);
+ else
+ ZONE_UNLOCK(zone);
- if (bucket == NULL) {
- return (0);
- }
+ return i;
+}
+
+static int
+zone_alloc_bucket(uma_zone_t zone, int flags)
+{
+ uma_bucket_t bucket;
+ int bflags;
+ int max;
#ifdef SMP
/*
@@ -2382,79 +2417,75 @@ zone_alloc_bucket(uma_zone_t zone, int f
* is done so that we don't allocate more memory than we really need.
*/
if (zone->uz_fills >= mp_ncpus)
- goto done;
+ return (0);
#endif
zone->uz_fills++;
+ max = zone->uz_count;
- max = MIN(bucket->ub_entries, zone->uz_count);
- /* Try to keep the buckets totally full */
- saved = bucket->ub_cnt;
- slab = NULL;
- keg = NULL;
- while (bucket->ub_cnt < max &&
- (slab = zone->uz_slab(zone, keg, flags)) != NULL) {
- keg = slab->us_keg;
- while (slab->us_freecount && bucket->ub_cnt < max) {
- bucket->ub_bucket[bucket->ub_cnt++] =
- slab_alloc_item(zone, slab);
- }
-
- /* Don't block on the next fill */
- flags |= M_NOWAIT;
+ /*
+ * Try this zone's free list first so we don't allocate extra buckets.
+ */
+ if ((bucket = LIST_FIRST(&zone->uz_free_bucket)) != NULL) {
+ KASSERT(bucket->ub_cnt == 0,
+ ("zone_alloc_bucket: Bucket on free list is not empty."));
+ LIST_REMOVE(bucket, ub_link);
+ ZONE_UNLOCK(zone);
+ } else {
+ bflags = (flags & ~M_ZERO);
+ if (zone->uz_flags & UMA_ZFLAG_CACHEONLY)
+ bflags |= M_NOVM;
+ ZONE_UNLOCK(zone);
+ bucket = bucket_alloc(zone->uz_count, bflags);
+ if (bucket == NULL)
+ goto out;
}
- if (slab)
- zone_relock(zone, keg);
+
+ max = MIN(bucket->ub_entries, max);
+ bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket,
+ max, flags);
/*
- * We unlock here because we need to call the zone's init.
- * It should be safe to unlock because the slab dealt with
- * above is already on the appropriate list within the keg
- * and the bucket we filled is not yet on any list, so we
- * own it.
+ * Initialize the memory if necessary.
*/
- if (zone->uz_init != NULL) {
+ if (bucket->ub_cnt != 0 && zone->uz_init != NULL) {
int i;
- ZONE_UNLOCK(zone);
- for (i = saved; i < bucket->ub_cnt; i++)
+ for (i = 0; i < bucket->ub_cnt; i++)
if (zone->uz_init(bucket->ub_bucket[i], zone->uz_size,
- origflags) != 0)
+ flags) != 0)
break;
/*
* If we couldn't initialize the whole bucket, put the
* rest back onto the freelist.
*/
if (i != bucket->ub_cnt) {
- int j;
-
- for (j = i; j < bucket->ub_cnt; j++) {
- zone_free_item(zone, bucket->ub_bucket[j],
- NULL, SKIP_FINI, 0);
+ zone->uz_release(zone->uz_arg, bucket->ub_bucket[i],
+ bucket->ub_cnt - i);
#ifdef INVARIANTS
- bucket->ub_bucket[j] = NULL;
+ bzero(&bucket->ub_bucket[i],
+ sizeof(void *) * (bucket->ub_cnt - i));
#endif
- }
bucket->ub_cnt = i;
}
- ZONE_LOCK(zone);
}
+out:
+ ZONE_LOCK(zone);
zone->uz_fills--;
- if (bucket->ub_cnt != 0) {
+ if (bucket != NULL && bucket->ub_cnt != 0) {
LIST_INSERT_HEAD(&zone->uz_full_bucket,
bucket, ub_link);
return (1);
}
-#ifdef SMP
-done:
-#endif
- bucket_free(bucket);
+ atomic_add_long(&zone->uz_fails, 1);
+ if (bucket != NULL)
+ bucket_free(bucket);
return (0);
}
/*
- * Allocates an item for an internal zone
+ * Allocates a single item from a zone.
*
* Arguments
* zone The zone to alloc for.
@@ -2469,7 +2500,6 @@ done:
static void *
zone_alloc_item(uma_zone_t zone, void *udata, int flags)
{
- uma_slab_t slab;
void *item;
item = NULL;
@@ -2477,20 +2507,9 @@ zone_alloc_item(uma_zone_t zone, void *u
#ifdef UMA_DEBUG_ALLOC
printf("INTERNAL: Allocating one item from %s(%p)\n", zone->uz_name,
zone);
#endif
- ZONE_LOCK(zone);
-
- slab = zone->uz_slab(zone, NULL, flags);
- if (slab == NULL) {
- zone->uz_fails++;
- ZONE_UNLOCK(zone);
- return (NULL);
- }
-
- item = slab_alloc_item(zone, slab);
-
- zone_relock(zone, slab->us_keg);
- zone->uz_allocs++;
- ZONE_UNLOCK(zone);
+ if (zone->uz_import(zone->uz_arg, &item, 1, flags) != 1)
+ goto fail;
+ atomic_add_long(&zone->uz_allocs, 1);
/*
* We have to call both the zone's init (not the keg's init)
@@ -2500,25 +2519,27 @@ zone_alloc_item(uma_zone_t zone, void *u
*/
if (zone->uz_init != NULL) {
if (zone->uz_init(item, zone->uz_size, flags) != 0) {
- zone_free_item(zone, item, udata, SKIP_FINI,
- ZFREE_STATFAIL | ZFREE_STATFREE);
- return (NULL);
+ zone_free_item(zone, item, udata, SKIP_FINI);
+ goto fail;
}
}
if (zone->uz_ctor != NULL) {
if (zone->uz_ctor(item, zone->uz_size, udata, flags) != 0) {
- zone_free_item(zone, item, udata, SKIP_DTOR,
- ZFREE_STATFAIL | ZFREE_STATFREE);
- return (NULL);
+ zone_free_item(zone, item, udata, SKIP_DTOR);
+ goto fail;
}
}
#ifdef INVARIANTS
- uma_dbg_alloc(zone, slab, item);
+ uma_dbg_alloc(zone, NULL, item);
#endif
if (flags & M_ZERO)
bzero(item, zone->uz_size);
return (item);
+
+fail:
+ atomic_add_long(&zone->uz_fails, 1);
+ return (NULL);
}
/* See uma.h */
@@ -2648,9 +2669,9 @@ zfree_start:
}
/* Since we have locked the zone we may as well send back our stats */
- zone->uz_allocs += cache->uc_allocs;
+ atomic_add_long(&zone->uz_allocs, cache->uc_allocs);
+ atomic_add_long(&zone->uz_frees, cache->uc_frees);
cache->uc_allocs = 0;
- zone->uz_frees += cache->uc_frees;
cache->uc_frees = 0;
bucket = cache->uc_freebucket;
@@ -2699,69 +2720,17 @@ zfree_start:
* If nothing else caught this, we'll just do an internal free.
*/
zfree_internal:
- zone_free_item(zone, item, udata, SKIP_DTOR, ZFREE_STATFREE);
+ zone_free_item(zone, item, udata, SKIP_DTOR);
return;
}
-/*
- * Frees an item to an INTERNAL zone or allocates a free bucket
- *
- * Arguments:
- * zone The zone to free to
- * item The item we're freeing
- * udata User supplied data for the dtor
- * skip Skip dtors and finis
- */
static void
-zone_free_item(uma_zone_t zone, void *item, void *udata,
- enum zfreeskip skip, int flags)
+slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item)
{
- uma_slab_t slab;
- uma_keg_t keg;
- uint8_t *mem;
uint8_t freei;
- int clearfull;
-#ifdef INVARIANTS
- if (skip == SKIP_NONE) {
- if (zone->uz_flags & UMA_ZONE_MALLOC)
- uma_dbg_free(zone, udata, item);
- else
- uma_dbg_free(zone, NULL, item);
- }
-#endif
- if (skip < SKIP_DTOR && zone->uz_dtor)
- zone->uz_dtor(item, zone->uz_size, udata);
-
- if (skip < SKIP_FINI && zone->uz_fini)
- zone->uz_fini(item, zone->uz_size);
-
- ZONE_LOCK(zone);
-
- if (flags & ZFREE_STATFAIL)
- zone->uz_fails++;
- if (flags & ZFREE_STATFREE)
- zone->uz_frees++;
-
- if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
- mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
- keg = zone_first_keg(zone); /* Must only be one. */
- if (zone->uz_flags & UMA_ZONE_HASH) {
- slab = hash_sfind(&keg->uk_hash, mem);
- } else {
- mem += keg->uk_pgoff;
- slab = (uma_slab_t)mem;
- }
- } else {
- /* This prevents redundant lookups via free(). */
- if ((zone->uz_flags & UMA_ZONE_MALLOC) && udata != NULL)
- slab = (uma_slab_t)udata;
- else
- slab = vtoslab((vm_offset_t)item);
- keg = slab->us_keg;
- keg_relock(keg, zone);
- }
+ mtx_assert(&keg->uk_lock, MA_OWNED);
MPASS(keg == slab->us_keg);
/* Do we need to remove from any lists? */
@@ -2780,31 +2749,94 @@ zone_free_item(uma_zone_t zone, void *it
/* Keg statistics. */
keg->uk_free++;
+}
+
+static void
+zone_release(uma_zone_t zone, void **bucket, int cnt)
+{
+ void *item;
+ uma_slab_t slab;
+ uma_keg_t keg;
+ uint8_t *mem;
+ int clearfull;
+ int i;
clearfull = 0;
- if (keg->uk_flags & UMA_ZFLAG_FULL) {
- if (keg->uk_pages < keg->uk_maxpages) {
- keg->uk_flags &= ~UMA_ZFLAG_FULL;
- clearfull = 1;
+ ZONE_LOCK(zone);
+ keg = zone_first_keg(zone);
+ for (i = 0; i < cnt; i++) {
+ item = bucket[i];
+ if (!(zone->uz_flags & UMA_ZONE_VTOSLAB)) {
+ mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK));
+ if (zone->uz_flags & UMA_ZONE_HASH) {
+ slab = hash_sfind(&keg->uk_hash, mem);
+ } else {
+ mem += keg->uk_pgoff;
+ slab = (uma_slab_t)mem;
+ }
+ } else {
+ slab = vtoslab((vm_offset_t)item);
+ if (slab->us_keg != keg) {
+ KEG_UNLOCK(keg);
+ keg = slab->us_keg;
+ KEG_LOCK(keg);
+ }
}
+ slab_free_item(keg, slab, item);
+ if (keg->uk_flags & UMA_ZFLAG_FULL) {
+ if (keg->uk_pages < keg->uk_maxpages) {
+ keg->uk_flags &= ~UMA_ZFLAG_FULL;
+ clearfull = 1;
+ }
- /*
- * We can handle one more allocation. Since we're
- * clearing ZFLAG_FULL, wake up all procs blocked
- * on pages. This should be uncommon, so keeping this
- * simple for now (rather than adding count of blocked
- * threads etc).
- */
- wakeup(keg);
+ /*
+ * We can handle one more allocation. Since we're
+ * clearing ZFLAG_FULL, wake up all procs blocked
+ * on pages. This should be uncommon, so keeping this
+ * simple for now (rather than adding count of blocked
+ * threads etc).
+ */
+ wakeup(keg);
+ }
}
+ zone_relock(zone, keg);
if (clearfull) {
- zone_relock(zone, keg);
zone->uz_flags &= ~UMA_ZFLAG_FULL;
wakeup(zone);
- ZONE_UNLOCK(zone);
- } else
- KEG_UNLOCK(keg);
+ }
+ ZONE_UNLOCK(zone);
+
+}
+
+/*
+ * Frees a single item to any zone.
+ *
+ * Arguments:
+ * zone The zone to free to
+ * item The item we're freeing
+ * udata User supplied data for the dtor
+ * skip Skip dtors and finis
+ */
+static void
+zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
+{
+#ifdef INVARIANTS
+ if (skip == SKIP_NONE) {
+ if (zone->uz_flags & UMA_ZONE_MALLOC)
+ uma_dbg_free(zone, udata, item);
+ else
+ uma_dbg_free(zone, NULL, item);
+ }
+#endif
+ if (skip < SKIP_DTOR && zone->uz_dtor)
+ zone->uz_dtor(item, zone->uz_size, udata);
+
+ if (skip < SKIP_FINI && zone->uz_fini)
+ zone->uz_fini(item, zone->uz_size);
+
+ atomic_add_long(&zone->uz_frees, 1);
+ zone->uz_release(zone->uz_arg, &item, 1);
}
/* See uma.h */
@@ -2813,8 +2845,10 @@ uma_zone_set_max(uma_zone_t zone, int ni
{
uma_keg_t keg;
- ZONE_LOCK(zone);
keg = zone_first_keg(zone);
+ if (keg == NULL)
+ return (0);
+ ZONE_LOCK(zone);
keg->uk_maxpages = (nitems / keg->uk_ipers) * keg->uk_ppera;
if (keg->uk_maxpages * keg->uk_ipers < nitems)
keg->uk_maxpages += keg->uk_ppera;
@@ -2831,8 +2865,10 @@ uma_zone_get_max(uma_zone_t zone)
int nitems;
uma_keg_t keg;
- ZONE_LOCK(zone);
keg = zone_first_keg(zone);
+ if (keg == NULL)
+ return (0);
+ ZONE_LOCK(zone);
nitems = keg->uk_maxpages * keg->uk_ipers;
ZONE_UNLOCK(zone);
@@ -2880,6 +2916,7 @@ uma_zone_set_init(uma_zone_t zone, uma_i
ZONE_LOCK(zone);
keg = zone_first_keg(zone);
+ KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
KASSERT(keg->uk_pages == 0,
("uma_zone_set_init on non-empty keg"));
keg->uk_init = uminit;
@@ -2894,6 +2931,7 @@ uma_zone_set_fini(uma_zone_t zone, uma_f
ZONE_LOCK(zone);
keg = zone_first_keg(zone);
+ KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
KASSERT(keg->uk_pages == 0,
("uma_zone_set_fini on non-empty keg"));
keg->uk_fini = fini;
@@ -2927,9 +2965,12 @@ uma_zone_set_zfini(uma_zone_t zone, uma_
void
uma_zone_set_freef(uma_zone_t zone, uma_free freef)
{
+ uma_keg_t keg;
ZONE_LOCK(zone);
- zone_first_keg(zone)->uk_freef = freef;
+ keg = zone_first_keg(zone);
+ KASSERT(keg != NULL, ("uma_zone_set_init: Invalid zone type"));
+ keg->uk_freef = freef;
ZONE_UNLOCK(zone);
}
@@ -2956,6 +2997,8 @@ uma_zone_reserve_kva(uma_zone_t zone, in
int pages;
keg = zone_first_keg(zone);
+ if (keg == NULL)
+ return (0);
pages = count / keg->uk_ipers;
if (pages * keg->uk_ipers < count)
@@ -2994,6 +3037,8 @@ uma_prealloc(uma_zone_t zone, int items)
uma_keg_t keg;
keg = zone_first_keg(zone);
+ if (keg == NULL)
+ return;
ZONE_LOCK(zone);
slabs = items / keg->uk_ipers;
if (slabs * keg->uk_ipers < items)
@@ -3083,8 +3128,7 @@ uma_large_malloc(int size, int wait)
slab->us_flags = flags | UMA_SLAB_MALLOC;
slab->us_size = size;
} else {
- zone_free_item(slabzone, slab, NULL, SKIP_NONE,
- ZFREE_STATFAIL | ZFREE_STATFREE);
+ zone_free_item(slabzone, slab, NULL, SKIP_NONE);
}
return (mem);
@@ -3095,7 +3139,7 @@ uma_large_free(uma_slab_t slab)
{
vsetobj((vm_offset_t)slab->us_data, kmem_object);
page_free(slab->us_data, slab->us_size, slab->us_flags);
- zone_free_item(slabzone, slab, NULL, SKIP_NONE, ZFREE_STATFREE);
+ zone_free_item(slabzone, slab, NULL, SKIP_NONE);
}
void
Modified: head/sys/vm/uma_int.h
==============================================================================
--- head/sys/vm/uma_int.h Mon Jun 17 03:32:27 2013 (r251825)
+++ head/sys/vm/uma_int.h Mon Jun 17 03:43:47 2013 (r251826)
@@ -296,14 +296,17 @@ struct uma_zone {
uma_ctor uz_ctor; /* Constructor for each allocation */
uma_dtor uz_dtor; /* Destructor */
uma_init uz_init; /* Initializer for each item */
- uma_fini uz_fini; /* Discards memory */
+ uma_fini uz_fini; /* Finalizer for each item. */
+ uma_import uz_import; /* Import new memory to cache. */
+ uma_release uz_release; /* Release memory from cache. */
+ void *uz_arg; /* Import/release argument. */
uint32_t uz_flags; /* Flags inherited from kegs */
uint32_t uz_size; /* Size inherited from kegs */
- uint64_t uz_allocs UMA_ALIGN; /* Total number of allocations */
- uint64_t uz_frees; /* Total number of frees */
- uint64_t uz_fails; /* Total number of alloc failures */
+ volatile u_long uz_allocs UMA_ALIGN; /* Total number of allocations */
+ volatile u_long uz_fails; /* Total number of alloc failures */
+ volatile u_long uz_frees; /* Total number of frees */
uint64_t uz_sleeps; /* Total number of alloc sleeps */
uint16_t uz_fills; /* Outstanding bucket fills */
uint16_t uz_count; /* Highest amount of items in bucket */
@@ -333,6 +336,13 @@ struct uma_zone {
#define UMA_ZFLAG_INHERIT (UMA_ZFLAG_INTERNAL |
UMA_ZFLAG_CACHEONLY | \
UMA_ZFLAG_BUCKET)
+static inline uma_keg_t
+zone_first_keg(uma_zone_t zone)
+{
+
+ return (LIST_FIRST(&zone->uz_kegs)->kl_keg);
+}
+
#undef UMA_ALIGN
#ifdef _KERNEL
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
