ipc/sem.c: optimize if semops fail

[pandora-kernel.git] / ipc / sem.c

diff --git a/ipc/sem.c b/ipc/sem.c
index 87c2b64..eac3f46 100644 (file)
--- a/ipc/sem.c
+++ b/ipc/sem.c
@@ -129,6 +129,7 @@ void sem_init_ns(struct ipc_namespace *ns)
 void sem_exit_ns(struct ipc_namespace *ns)
 {
        free_ipcs(ns, &sem_ids(ns), freeary);
+       idr_destroy(&ns->ids[IPC_SEM_IDS].ipcs_idr);
 }
 #endif
 
@@ -397,63 +398,61 @@ undo:
        return result;
 }
 
+/*
+ * Wake up a process waiting on the sem queue with a given error.
+ * The queue is invalid (may not be accessed) after the function returns.
+ */
+static void wake_up_sem_queue(struct sem_queue *q, int error)
+{
+       /*
+        * Hold preempt off so that we don't get preempted and have the
+        * wakee busy-wait until we're scheduled back on. We're holding
+        * locks here so it may not strictly be needed, however if the
+        * locks become preemptible then this prevents such a problem.
+        */
+       preempt_disable();
+       q->status = IN_WAKEUP;
+       wake_up_process(q->sleeper);
+       /* hands-off: q can disappear immediately after writing q->status. */
+       smp_wmb();
+       q->status = error;
+       preempt_enable();
+}
+
 /* Go through the pending queue for the indicated semaphore
  * looking for tasks that can be completed.
  */
 static void update_queue (struct sem_array * sma)
 {
-       int error;
-       struct sem_queue * q;
+       struct sem_queue *q, *tq;
+
+again:
+       list_for_each_entry_safe(q, tq, &sma->sem_pending, list) {
+               int error;
+               int alter;
 
-       q = list_entry(sma->sem_pending.next, struct sem_queue, list);
-       while (&q->list != &sma->sem_pending) {
                error = try_atomic_semop(sma, q->sops, q->nsops,
                                         q->undo, q->pid);
 
                /* Does q->sleeper still need to sleep? */
-               if (error <= 0) {
-                       struct sem_queue *n;
-
-                       /*
-                        * Continue scanning. The next operation
-                        * that must be checked depends on the type of the
-                        * completed operation:
-                        * - if the operation modified the array, then
-                        *   restart from the head of the queue and
-                        *   check for threads that might be waiting
-                        *   for semaphore values to become 0.
-                        * - if the operation didn't modify the array,
-                        *   then just continue.
-                        * The order of list_del() and reading ->next
-                        * is crucial: In the former case, the list_del()
-                        * must be done first [because we might be the
-                        * first entry in ->sem_pending], in the latter
-                        * case the list_del() must be done last
-                        * [because the list is invalid after the list_del()]
-                        */
-                       if (q->alter) {
-                               list_del(&q->list);
-                               n = list_entry(sma->sem_pending.next,
-                                               struct sem_queue, list);
-                       } else {
-                               n = list_entry(q->list.next, struct sem_queue,
-                                               list);
-                               list_del(&q->list);
-                       }
+               if (error > 0)
+                       continue;
 
-                       /* wake up the waiting thread */
-                       q->status = IN_WAKEUP;
+               list_del(&q->list);
 
-                       wake_up_process(q->sleeper);
-                       /* hands-off: q will disappear immediately after
-                        * writing q->status.
-                        */
-                       smp_wmb();
-                       q->status = error;
-                       q = n;
-               } else {
-                       q = list_entry(q->list.next, struct sem_queue, list);
-               }
+               /*
+                * The next operation that must be checked depends on the type
+                * of the completed operation:
+                * - if the operation modified the array, then restart from the
+                *   head of the queue and check for threads that might be
+                *   waiting for the new semaphore values.
+                * - if the operation didn't modify the array, then just
+                *   continue.
+                */
+               alter = q->alter;
+               wake_up_sem_queue(q, error);
+               if (alter && !error)
+                       goto again;
        }
 }
 
@@ -535,10 +534,7 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
        list_for_each_entry_safe(q, tq, &sma->sem_pending, list) {
                list_del(&q->list);
 
-               q->status = IN_WAKEUP;
-               wake_up_process(q->sleeper); /* doesn't sleep */
-               smp_wmb();
-               q->status = -EIDRM;     /* hands-off q */
+               wake_up_sem_queue(q, -EIDRM);
        }
 
        /* Remove the semaphore set from the IDR */
@@ -961,17 +957,31 @@ static inline int get_undo_list(struct sem_undo_list **undo_listp)
        return 0;
 }
 
-static struct sem_undo *lookup_undo(struct sem_undo_list *ulp, int semid)
+static struct sem_undo *__lookup_undo(struct sem_undo_list *ulp, int semid)
 {
-       struct sem_undo *walk;
+       struct sem_undo *un;
 
-       list_for_each_entry_rcu(walk, &ulp->list_proc, list_proc) {
-               if (walk->semid == semid)
-                       return walk;
+       list_for_each_entry_rcu(un, &ulp->list_proc, list_proc) {
+               if (un->semid == semid)
+                       return un;
        }
        return NULL;
 }
 
+static struct sem_undo *lookup_undo(struct sem_undo_list *ulp, int semid)
+{
+       struct sem_undo *un;
+
+       assert_spin_locked(&ulp->lock);
+
+       un = __lookup_undo(ulp, semid);
+       if (un) {
+               list_del_rcu(&un->list_proc);
+               list_add_rcu(&un->list_proc, &ulp->list_proc);
+       }
+       return un;
+}
+
 /**
  * find_alloc_undo - Lookup (and if not present create) undo array
  * @ns: namespace
@@ -1307,7 +1317,7 @@ void exit_sem(struct task_struct *tsk)
                if (IS_ERR(sma))
                        continue;
 
-               un = lookup_undo(ulp, semid);
+               un = __lookup_undo(ulp, semid);
                if (un == NULL) {
                        /* exit_sem raced with IPC_RMID+semget() that created
                         * exactly the same semid. Nothing to do.