git.openpandora.org
/
pandora-kernel.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
slub: use raw_cpu_inc for incrementing statistics
[pandora-kernel.git]
/
mm
/
slub.c
diff --git
a/mm/slub.c
b/mm/slub.c
index
25f14ad
..
f620bbf
100644
(file)
--- a/
mm/slub.c
+++ b/
mm/slub.c
@@
-224,7
+224,11
@@
static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { }
static inline void stat(const struct kmem_cache *s, enum stat_item si)
{
#ifdef CONFIG_SLUB_STATS
static inline void stat(const struct kmem_cache *s, enum stat_item si)
{
#ifdef CONFIG_SLUB_STATS
- __this_cpu_inc(s->cpu_slab->stat[si]);
+ /*
+ * The rmw is racy on a preemptible kernel but this is acceptable, so
+ * avoid this_cpu_add()'s irq-disable overhead.
+ */
+ raw_cpu_inc(s->cpu_slab->stat[si]);
#endif
}
#endif
}
@@
-1684,8
+1688,8
@@
static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
return NULL;
do {
return NULL;
do {
- cpuset_mems_cookie =
get_mems_allowed
();
- zonelist = node_zonelist(slab_node(), flags);
+ cpuset_mems_cookie =
read_mems_allowed_begin
();
+ zonelist = node_zonelist(
mempolicy_
slab_node(), flags);
for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
struct kmem_cache_node *n;
for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
struct kmem_cache_node *n;
@@
-1696,19
+1700,17
@@
static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
object = get_partial_node(s, n, c, flags);
if (object) {
/*
object = get_partial_node(s, n, c, flags);
if (object) {
/*
- * Return the object even if
- * put_mems_allowed indicated that
- * the cpuset mems_allowed was
- * updated in parallel. It's a
- * harmless race between the alloc
- * and the cpuset update.
+ * Don't check read_mems_allowed_retry()
+ * here - if mems_allowed was updated in
+ * parallel, that was a harmless race
+ * between allocation and the cpuset
+ * update
*/
*/
- put_mems_allowed(cpuset_mems_cookie);
return object;
}
}
}
return object;
}
}
}
- } while (
!put_mems_allowed
(cpuset_mems_cookie));
+ } while (
read_mems_allowed_retry
(cpuset_mems_cookie));
#endif
return NULL;
}
#endif
return NULL;
}
@@
-3239,8
+3241,9
@@
int __kmem_cache_shutdown(struct kmem_cache *s)
if (!rc) {
/*
if (!rc) {
/*
- * We do the same lock strategy around sysfs_slab_add, see
- * __kmem_cache_create. Because this is pretty much the last
+ * Since slab_attr_store may take the slab_mutex, we should
+ * release the lock while removing the sysfs entry in order to
+ * avoid a deadlock. Because this is pretty much the last
* operation we do and the lock will be released shortly after
* that in slab_common.c, we could just move sysfs_slab_remove
* to a later point in common code. We should do that when we
* operation we do and the lock will be released shortly after
* that in slab_common.c, we could just move sysfs_slab_remove
* to a later point in common code. We should do that when we
@@
-3686,6
+3689,9
@@
static int slab_unmergeable(struct kmem_cache *s)
if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
return 1;
if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
return 1;
+ if (!is_root_cache(s))
+ return 1;
+
if (s->ctor)
return 1;
if (s->ctor)
return 1;
@@
-3698,9
+3704,8
@@
static int slab_unmergeable(struct kmem_cache *s)
return 0;
}
return 0;
}
-static struct kmem_cache *find_mergeable(struct mem_cgroup *memcg, size_t size,
- size_t align, unsigned long flags, const char *name,
- void (*ctor)(void *))
+static struct kmem_cache *find_mergeable(size_t size, size_t align,
+ unsigned long flags, const char *name, void (*ctor)(void *))
{
struct kmem_cache *s;
{
struct kmem_cache *s;
@@
-3723,7
+3728,7
@@
static struct kmem_cache *find_mergeable(struct mem_cgroup *memcg, size_t size,
continue;
if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
continue;
if ((flags & SLUB_MERGE_SAME) != (s->flags & SLUB_MERGE_SAME))
-
continue;
+ continue;
/*
* Check if alignment is compatible.
* Courtesy of Adrian Drzewiecki
/*
* Check if alignment is compatible.
* Courtesy of Adrian Drzewiecki
@@
-3734,23
+3739,24
@@
static struct kmem_cache *find_mergeable(struct mem_cgroup *memcg, size_t size,
if (s->size - size >= sizeof(void *))
continue;
if (s->size - size >= sizeof(void *))
continue;
- if (!cache_match_memcg(s, memcg))
- continue;
-
return s;
}
return NULL;
}
struct kmem_cache *
return s;
}
return NULL;
}
struct kmem_cache *
-__kmem_cache_alias(
struct mem_cgroup *memcg, const char *name, size_t size
,
-
size_t align,
unsigned long flags, void (*ctor)(void *))
+__kmem_cache_alias(
const char *name, size_t size, size_t align
,
+ unsigned long flags, void (*ctor)(void *))
{
struct kmem_cache *s;
{
struct kmem_cache *s;
- s = find_mergeable(
memcg,
size, align, flags, name, ctor);
+ s = find_mergeable(size, align, flags, name, ctor);
if (s) {
if (s) {
+ int i;
+ struct kmem_cache *c;
+
s->refcount++;
s->refcount++;
+
/*
* Adjust the object sizes so that we clear
* the complete object on kzalloc.
/*
* Adjust the object sizes so that we clear
* the complete object on kzalloc.
@@
-3758,6
+3764,15
@@
__kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
s->object_size = max(s->object_size, (int)size);
s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
s->object_size = max(s->object_size, (int)size);
s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
+ for_each_memcg_cache_index(i) {
+ c = cache_from_memcg_idx(s, i);
+ if (!c)
+ continue;
+ c->object_size = s->object_size;
+ c->inuse = max_t(int, c->inuse,
+ ALIGN(size, sizeof(void *)));
+ }
+
if (sysfs_slab_alias(s, name)) {
s->refcount--;
s = NULL;
if (sysfs_slab_alias(s, name)) {
s->refcount--;
s = NULL;
@@
-3780,10
+3795,7
@@
int __kmem_cache_create(struct kmem_cache *s, unsigned long flags)
return 0;
memcg_propagate_slab_attrs(s);
return 0;
memcg_propagate_slab_attrs(s);
- mutex_unlock(&slab_mutex);
err = sysfs_slab_add(s);
err = sysfs_slab_add(s);
- mutex_lock(&slab_mutex);
-
if (err)
kmem_cache_close(s);
if (err)
kmem_cache_close(s);
@@
-5130,6
+5142,15
@@
static const struct kset_uevent_ops slab_uevent_ops = {
static struct kset *slab_kset;
static struct kset *slab_kset;
+static inline struct kset *cache_kset(struct kmem_cache *s)
+{
+#ifdef CONFIG_MEMCG_KMEM
+ if (!is_root_cache(s))
+ return s->memcg_params->root_cache->memcg_kset;
+#endif
+ return slab_kset;
+}
+
#define ID_STR_LENGTH 64
/* Create a unique string id for a slab cache:
#define ID_STR_LENGTH 64
/* Create a unique string id for a slab cache:
@@
-5195,26
+5216,39
@@
static int sysfs_slab_add(struct kmem_cache *s)
name = create_unique_id(s);
}
name = create_unique_id(s);
}
- s->kobj.kset =
slab_kset
;
+ s->kobj.kset =
cache_kset(s)
;
err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
- if (err) {
- kobject_put(&s->kobj);
- return err;
- }
+ if (err)
+ goto out_put_kobj;
err = sysfs_create_group(&s->kobj, &slab_attr_group);
err = sysfs_create_group(&s->kobj, &slab_attr_group);
- if (err) {
- kobject_del(&s->kobj);
- kobject_put(&s->kobj);
- return err;
+ if (err)
+ goto out_del_kobj;
+
+#ifdef CONFIG_MEMCG_KMEM
+ if (is_root_cache(s)) {
+ s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj);
+ if (!s->memcg_kset) {
+ err = -ENOMEM;
+ goto out_del_kobj;
+ }
}
}
+#endif
+
kobject_uevent(&s->kobj, KOBJ_ADD);
if (!unmergeable) {
/* Setup first alias */
sysfs_slab_alias(s, s->name);
kobject_uevent(&s->kobj, KOBJ_ADD);
if (!unmergeable) {
/* Setup first alias */
sysfs_slab_alias(s, s->name);
- kfree(name);
}
}
- return 0;
+out:
+ if (!unmergeable)
+ kfree(name);
+ return err;
+out_del_kobj:
+ kobject_del(&s->kobj);
+out_put_kobj:
+ kobject_put(&s->kobj);
+ goto out;
}
static void sysfs_slab_remove(struct kmem_cache *s)
}
static void sysfs_slab_remove(struct kmem_cache *s)
@@
-5226,6
+5260,9
@@
static void sysfs_slab_remove(struct kmem_cache *s)
*/
return;
*/
return;
+#ifdef CONFIG_MEMCG_KMEM
+ kset_unregister(s->memcg_kset);
+#endif
kobject_uevent(&s->kobj, KOBJ_REMOVE);
kobject_del(&s->kobj);
kobject_put(&s->kobj);
kobject_uevent(&s->kobj, KOBJ_REMOVE);
kobject_del(&s->kobj);
kobject_put(&s->kobj);