drivers/gpu/drm/amd/scheduler/gpu_scheduler.c

   1 /*
   2  * Copyright 2015 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24 #include <linux/kthread.h>
  25 #include <linux/wait.h>
  26 #include <linux/sched.h>
  27 #include <drm/drmP.h>
  28 #include "gpu_scheduler.h"
  29
  30 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
  31
  32 /* Initialize a given run queue struct */
  33 static void amd_sched_rq_init(struct amd_sched_rq *rq)
  34 {
  35         spin_lock_init(&rq->lock);
  36         INIT_LIST_HEAD(&rq->entities);
  37         rq->current_entity = NULL;
  38 }
  39
  40 static void amd_sched_rq_add_entity(struct amd_sched_rq *rq,
  41                                     struct amd_sched_entity *entity)
  42 {
  43         spin_lock(&rq->lock);
  44         list_add_tail(&entity->list, &rq->entities);
  45         spin_unlock(&rq->lock);
  46 }
  47
  48 static void amd_sched_rq_remove_entity(struct amd_sched_rq *rq,
  49                                        struct amd_sched_entity *entity)
  50 {
  51         spin_lock(&rq->lock);
  52         list_del_init(&entity->list);
  53         if (rq->current_entity == entity)
  54                 rq->current_entity = NULL;
  55         spin_unlock(&rq->lock);
  56 }
  57
  58 /**
  59  * Select next entity from a specified run queue with round robin policy.
  60  * It could return the same entity as current one if current is the only
  61  * available one in the queue. Return NULL if nothing available.
  62  */
  63 static struct amd_sched_entity *
  64 amd_sched_rq_select_entity(struct amd_sched_rq *rq)
  65 {
  66         struct amd_sched_entity *entity;
  67
  68         spin_lock(&rq->lock);
  69
  70         entity = rq->current_entity;
  71         if (entity) {
  72                 list_for_each_entry_continue(entity, &rq->entities, list) {
  73                         if (!kfifo_is_empty(&entity->job_queue)) {
  74                                 rq->current_entity = entity;
  75                                 spin_unlock(&rq->lock);
  76                                 return rq->current_entity;
  77                         }
  78                 }
  79         }
  80
  81         list_for_each_entry(entity, &rq->entities, list) {
  82
  83                 if (!kfifo_is_empty(&entity->job_queue)) {
  84                         rq->current_entity = entity;
  85                         spin_unlock(&rq->lock);
  86                         return rq->current_entity;
  87                 }
  88
  89                 if (entity == rq->current_entity)
  90                         break;
  91         }
  92
  93         spin_unlock(&rq->lock);
  94
  95         return NULL;
  96 }
  97
  98 /**
  99  * Init a context entity used by scheduler when submit to HW ring.
 100  *
 101  * @sched       The pointer to the scheduler
 102  * @entity      The pointer to a valid amd_sched_entity
 103  * @rq          The run queue this entity belongs
 104  * @kernel      If this is an entity for the kernel
 105  * @jobs        The max number of jobs in the job queue
 106  *
 107  * return 0 if succeed. negative error code on failure
 108 */
 109 int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
 110                           struct amd_sched_entity *entity,
 111                           struct amd_sched_rq *rq,
 112                           uint32_t jobs)
 113 {
 114         if (!(sched && entity && rq))
 115                 return -EINVAL;
 116
 117         memset(entity, 0, sizeof(struct amd_sched_entity));
 118         entity->belongto_rq = rq;
 119         entity->scheduler = sched;
 120         entity->fence_context = fence_context_alloc(1);
 121         if(kfifo_alloc(&entity->job_queue,
 122                        jobs * sizeof(void *),
 123                        GFP_KERNEL))
 124                 return -EINVAL;
 125
 126         spin_lock_init(&entity->queue_lock);
 127         atomic_set(&entity->fence_seq, 0);
 128
 129         /* Add the entity to the run queue */
 130         amd_sched_rq_add_entity(rq, entity);
 131         return 0;
 132 }
 133
 134 /**
 135  * Query if entity is initialized
 136  *
 137  * @sched       Pointer to scheduler instance
 138  * @entity      The pointer to a valid scheduler entity
 139  *
 140  * return true if entity is initialized, false otherwise
 141 */
 142 static bool amd_sched_entity_is_initialized(struct amd_gpu_scheduler *sched,
 143                                             struct amd_sched_entity *entity)
 144 {
 145         return entity->scheduler == sched &&
 146                 entity->belongto_rq != NULL;
 147 }
 148
 149 /**
 150  * Check if entity is idle
 151  *
 152  * @entity      The pointer to a valid scheduler entity
 153  *
 154  * Return true if entity don't has any unscheduled jobs.
 155  */
 156 static bool amd_sched_entity_is_idle(struct amd_sched_entity *entity)
 157 {
 158         rmb();
 159         if (kfifo_is_empty(&entity->job_queue))
 160                 return true;
 161
 162         return false;
 163 }
 164
 165 /**
 166  * Destroy a context entity
 167  *
 168  * @sched       Pointer to scheduler instance
 169  * @entity      The pointer to a valid scheduler entity
 170  *
 171  * Cleanup and free the allocated resources.
 172  */
 173 void amd_sched_entity_fini(struct amd_gpu_scheduler *sched,
 174                            struct amd_sched_entity *entity)
 175 {
 176         struct amd_sched_rq *rq = entity->belongto_rq;
 177
 178         if (!amd_sched_entity_is_initialized(sched, entity))
 179                 return;
 180
 181         /**
 182          * The client will not queue more IBs during this fini, consume existing
 183          * queued IBs
 184         */
 185         wait_event(sched->job_scheduled, amd_sched_entity_is_idle(entity));
 186
 187         amd_sched_rq_remove_entity(rq, entity);
 188         kfifo_free(&entity->job_queue);
 189 }
 190
 191 /**
 192  * Helper to submit a job to the job queue
 193  *
 194  * @job         The pointer to job required to submit
 195  *
 196  * Returns true if we could submit the job.
 197  */
 198 static bool amd_sched_entity_in(struct amd_sched_job *job)
 199 {
 200         struct amd_sched_entity *entity = job->s_entity;
 201         bool added, first = false;
 202
 203         spin_lock(&entity->queue_lock);
 204         added = kfifo_in(&entity->job_queue, &job, sizeof(job)) == sizeof(job);
 205
 206         if (added && kfifo_len(&entity->job_queue) == sizeof(job))
 207                 first = true;
 208
 209         spin_unlock(&entity->queue_lock);
 210
 211         /* first job wakes up scheduler */
 212         if (first)
 213                 amd_sched_wakeup(job->sched);
 214
 215         return added;
 216 }
 217
 218 /**
 219  * Submit a job to the job queue
 220  *
 221  * @job         The pointer to job required to submit
 222  *
 223  * Returns 0 for success, negative error code otherwise.
 224  */
 225 int amd_sched_entity_push_job(struct amd_sched_job *sched_job)
 226 {
 227         struct amd_sched_entity *entity = sched_job->s_entity;
 228         struct amd_sched_fence *fence = amd_sched_fence_create(
 229                 entity, sched_job->owner);
 230         int r;
 231
 232         if (!fence)
 233                 return -ENOMEM;
 234
 235         fence_get(&fence->base);
 236         sched_job->s_fence = fence;
 237
 238         r = wait_event_interruptible(entity->scheduler->job_scheduled,
 239                                      amd_sched_entity_in(sched_job));
 240
 241         return r;
 242 }
 243
 244 /**
 245  * Return ture if we can push more jobs to the hw.
 246  */
 247 static bool amd_sched_ready(struct amd_gpu_scheduler *sched)
 248 {
 249         return atomic_read(&sched->hw_rq_count) <
 250                 sched->hw_submission_limit;
 251 }
 252
 253 /**
 254  * Wake up the scheduler when it is ready
 255  */
 256 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched)
 257 {
 258         if (amd_sched_ready(sched))
 259                 wake_up_interruptible(&sched->wake_up_worker);
 260 }
 261
 262 /**
 263  * Select next entity containing real IB submissions
 264 */
 265 static struct amd_sched_entity *
 266 amd_sched_select_context(struct amd_gpu_scheduler *sched)
 267 {
 268         struct amd_sched_entity *tmp;
 269
 270         if (!amd_sched_ready(sched))
 271                 return NULL;
 272
 273         /* Kernel run queue has higher priority than normal run queue*/
 274         tmp = amd_sched_rq_select_entity(&sched->kernel_rq);
 275         if (tmp == NULL)
 276                 tmp = amd_sched_rq_select_entity(&sched->sched_rq);
 277
 278         return tmp;
 279 }
 280
 281 static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
 282 {
 283         struct amd_sched_job *sched_job =
 284                 container_of(cb, struct amd_sched_job, cb);
 285         struct amd_gpu_scheduler *sched;
 286
 287         sched = sched_job->sched;
 288         amd_sched_fence_signal(sched_job->s_fence);
 289         atomic_dec(&sched->hw_rq_count);
 290         fence_put(&sched_job->s_fence->base);
 291         sched->ops->process_job(sched_job);
 292         wake_up_interruptible(&sched->wake_up_worker);
 293 }
 294
 295 static int amd_sched_main(void *param)
 296 {
 297         struct sched_param sparam = {.sched_priority = 1};
 298         struct amd_gpu_scheduler *sched = (struct amd_gpu_scheduler *)param;
 299         int r;
 300
 301         sched_setscheduler(current, SCHED_FIFO, &sparam);
 302
 303         while (!kthread_should_stop()) {
 304                 struct amd_sched_entity *c_entity = NULL;
 305                 struct amd_sched_job *job;
 306                 struct fence *fence;
 307
 308                 wait_event_interruptible(sched->wake_up_worker,
 309                         kthread_should_stop() ||
 310                         (c_entity = amd_sched_select_context(sched)));
 311
 312                 if (!c_entity)
 313                         continue;
 314
 315                 r = kfifo_out(&c_entity->job_queue, &job, sizeof(void *));
 316                 if (r != sizeof(void *))
 317                         continue;
 318                 atomic_inc(&sched->hw_rq_count);
 319
 320                 fence = sched->ops->run_job(job);
 321                 if (fence) {
 322                         r = fence_add_callback(fence, &job->cb,
 323                                                amd_sched_process_job);
 324                         if (r == -ENOENT)
 325                                 amd_sched_process_job(fence, &job->cb);
 326                         else if (r)
 327                                 DRM_ERROR("fence add callback failed (%d)\n", r);
 328                         fence_put(fence);
 329                 }
 330
 331                 wake_up(&sched->job_scheduled);
 332         }
 333         return 0;
 334 }
 335
 336 /**
 337  * Create a gpu scheduler
 338  *
 339  * @ops                 The backend operations for this scheduler.
 340  * @ring                The the ring id for the scheduler.
 341  * @hw_submissions      Number of hw submissions to do.
 342  *
 343  * Return the pointer to scheduler for success, otherwise return NULL
 344 */
 345 struct amd_gpu_scheduler *amd_sched_create(struct amd_sched_backend_ops *ops,
 346                                            unsigned ring, unsigned hw_submission,
 347                                            void *priv)
 348 {
 349         struct amd_gpu_scheduler *sched;
 350
 351         sched = kzalloc(sizeof(struct amd_gpu_scheduler), GFP_KERNEL);
 352         if (!sched)
 353                 return NULL;
 354
 355         sched->ops = ops;
 356         sched->ring_id = ring;
 357         sched->hw_submission_limit = hw_submission;
 358         sched->priv = priv;
 359         snprintf(sched->name, sizeof(sched->name), "amdgpu[%d]", ring);
 360         amd_sched_rq_init(&sched->sched_rq);
 361         amd_sched_rq_init(&sched->kernel_rq);
 362
 363         init_waitqueue_head(&sched->wake_up_worker);
 364         init_waitqueue_head(&sched->job_scheduled);
 365         atomic_set(&sched->hw_rq_count, 0);
 366         /* Each scheduler will run on a seperate kernel thread */
 367         sched->thread = kthread_run(amd_sched_main, sched, sched->name);
 368         if (IS_ERR(sched->thread)) {
 369                 DRM_ERROR("Failed to create scheduler for id %d.\n", ring);
 370                 kfree(sched);
 371                 return NULL;
 372         }
 373
 374         return sched;
 375 }
 376
 377 /**
 378  * Destroy a gpu scheduler
 379  *
 380  * @sched       The pointer to the scheduler
 381  *
 382  * return 0 if succeed. -1 if failed.
 383  */
 384 int amd_sched_destroy(struct amd_gpu_scheduler *sched)
 385 {
 386         kthread_stop(sched->thread);
 387         kfree(sched);
 388         return  0;
 389 }