fs: dcache scale hash

Add a new lock, dcache_hash_lock, to protect the dcache hash table from
concurrent modification. d_hash is also protected by d_lock.

Signed-off-by: Nick Piggin <npiggin@kernel.dk>

diff --git a/fs/dcache.c b/fs/dcache.c
index 61beb40..1e124d4 100644 (file)
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -35,10 +35,24 @@
 #include <linux/hardirq.h>
 #include "internal.h"
 
+/*
+ * Usage:
+ * dcache_hash_lock protects dcache hash table, s_anon lists
+ *
+ * Ordering:
+ * dcache_lock
+ *   dentry->d_lock
+ *     dcache_hash_lock
+ *
+ * if (dentry1 < dentry2)
+ *   dentry1->d_lock
+ *     dentry2->d_lock
+ */
 int sysctl_vfs_cache_pressure __read_mostly = 100;
 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
 
- __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock);
+static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock);
+__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock);
 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
 
 EXPORT_SYMBOL(dcache_lock);
@@ -196,6 +210,42 @@ static struct dentry *d_kill(struct dentry *dentry)
        return parent;
 }
 
+/**
+ * d_drop - drop a dentry
+ * @dentry: dentry to drop
+ *
+ * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
+ * be found through a VFS lookup any more. Note that this is different from
+ * deleting the dentry - d_delete will try to mark the dentry negative if
+ * possible, giving a successful _negative_ lookup, while d_drop will
+ * just make the cache lookup fail.
+ *
+ * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
+ * reason (NFS timeouts or autofs deletes).
+ *
+ * __d_drop requires dentry->d_lock.
+ */
+void __d_drop(struct dentry *dentry)
+{
+       if (!(dentry->d_flags & DCACHE_UNHASHED)) {
+               dentry->d_flags |= DCACHE_UNHASHED;
+               spin_lock(&dcache_hash_lock);
+               hlist_del_rcu(&dentry->d_hash);
+               spin_unlock(&dcache_hash_lock);
+       }
+}
+EXPORT_SYMBOL(__d_drop);
+
+void d_drop(struct dentry *dentry)
+{
+       spin_lock(&dcache_lock);
+       spin_lock(&dentry->d_lock);
+       __d_drop(dentry);
+       spin_unlock(&dentry->d_lock);
+       spin_unlock(&dcache_lock);
+}
+EXPORT_SYMBOL(d_drop);
+
 /* 
  * This is dput
  *
@@ -1199,7 +1249,9 @@ struct dentry *d_obtain_alias(struct inode *inode)
        tmp->d_flags |= DCACHE_DISCONNECTED;
        tmp->d_flags &= ~DCACHE_UNHASHED;
        list_add(&tmp->d_alias, &inode->i_dentry);
+       spin_lock(&dcache_hash_lock);
        hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon);
+       spin_unlock(&dcache_hash_lock);
        spin_unlock(&tmp->d_lock);
 
        spin_unlock(&dcache_lock);
@@ -1585,7 +1637,9 @@ void d_rehash(struct dentry * entry)
 {
        spin_lock(&dcache_lock);
        spin_lock(&entry->d_lock);
+       spin_lock(&dcache_hash_lock);
        _d_rehash(entry);
+       spin_unlock(&dcache_hash_lock);
        spin_unlock(&entry->d_lock);
        spin_unlock(&dcache_lock);
 }
@@ -1692,8 +1746,6 @@ static void switch_names(struct dentry *dentry, struct dentry *target)
  */
 static void d_move_locked(struct dentry * dentry, struct dentry * target)
 {
-       struct hlist_head *list;
-
        if (!dentry->d_inode)
                printk(KERN_WARNING "VFS: moving negative dcache entry\n");
 
@@ -1710,14 +1762,11 @@ static void d_move_locked(struct dentry * dentry, struct dentry * target)
        }
 
        /* Move the dentry to the target hash queue, if on different bucket */
-       if (d_unhashed(dentry))
-               goto already_unhashed;
-
-       hlist_del_rcu(&dentry->d_hash);
-
-already_unhashed:
-       list = d_hash(target->d_parent, target->d_name.hash);
-       __d_rehash(dentry, list);
+       spin_lock(&dcache_hash_lock);
+       if (!d_unhashed(dentry))
+               hlist_del_rcu(&dentry->d_hash);
+       __d_rehash(dentry, d_hash(target->d_parent, target->d_name.hash));
+       spin_unlock(&dcache_hash_lock);
 
        /* Unhash the target: dput() will then get rid of it */
        __d_drop(target);
@@ -1914,7 +1963,9 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode)
 found_lock:
        spin_lock(&actual->d_lock);
 found:
+       spin_lock(&dcache_hash_lock);
        _d_rehash(actual);
+       spin_unlock(&dcache_hash_lock);
        spin_unlock(&actual->d_lock);
        spin_unlock(&dcache_lock);
 out_nolock:

diff --git a/include/linux/dcache.h b/include/linux/dcache.h
index cea27df..2feb624 100644 (file)
--- a/include/linux/dcache.h
+++ b/include/linux/dcache.h
@@ -184,39 +184,6 @@ struct dentry_operations {
 extern spinlock_t dcache_lock;
 extern seqlock_t rename_lock;
 
-/**
- * d_drop - drop a dentry
- * @dentry: dentry to drop
- *
- * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
- * be found through a VFS lookup any more. Note that this is different from
- * deleting the dentry - d_delete will try to mark the dentry negative if
- * possible, giving a successful _negative_ lookup, while d_drop will
- * just make the cache lookup fail.
- *
- * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
- * reason (NFS timeouts or autofs deletes).
- *
- * __d_drop requires dentry->d_lock.
- */
-
-static inline void __d_drop(struct dentry *dentry)
-{
-       if (!(dentry->d_flags & DCACHE_UNHASHED)) {
-               dentry->d_flags |= DCACHE_UNHASHED;
-               hlist_del_rcu(&dentry->d_hash);
-       }
-}
-
-static inline void d_drop(struct dentry *dentry)
-{
-       spin_lock(&dcache_lock);
-       spin_lock(&dentry->d_lock);
-       __d_drop(dentry);
-       spin_unlock(&dentry->d_lock);
-       spin_unlock(&dcache_lock);
-}
-
 static inline int dname_external(struct dentry *dentry)
 {
        return dentry->d_name.name != dentry->d_iname;
@@ -228,6 +195,8 @@ static inline int dname_external(struct dentry *dentry)
 extern void d_instantiate(struct dentry *, struct inode *);
 extern struct dentry * d_instantiate_unique(struct dentry *, struct inode *);
 extern struct dentry * d_materialise_unique(struct dentry *, struct inode *);
+extern void __d_drop(struct dentry *dentry);
+extern void d_drop(struct dentry *dentry);
 extern void d_delete(struct dentry *);
 
 /* allocate/de-allocate */