drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/slab.h>
  25 #include <linux/list.h>
  26 #include <linux/types.h>
  27 #include <linux/printk.h>
  28 #include <linux/bitops.h>
  29 #include "kfd_priv.h"
  30 #include "kfd_device_queue_manager.h"
  31 #include "kfd_mqd_manager.h"
  32 #include "cik_regs.h"
  33 #include "kfd_kernel_queue.h"
  34 #include "../../radeon/cik_reg.h"
  35
  36 /* Size of the per-pipe EOP queue */
  37 #define CIK_HPD_EOP_BYTES_LOG2 11
  38 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
  39
  40 static bool is_mem_initialized;
  41
  42 static int init_memory(struct device_queue_manager *dqm);
  43 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
  44                                         unsigned int pasid, unsigned int vmid);
  45
  46 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
  47                                         struct queue *q,
  48                                         struct qcm_process_device *qpd);
  49 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
  50 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock);
  51
  52
  53 static inline unsigned int get_pipes_num(struct device_queue_manager *dqm)
  54 {
  55         BUG_ON(!dqm || !dqm->dev);
  56         return dqm->dev->shared_resources.compute_pipe_count;
  57 }
  58
  59 static inline unsigned int get_first_pipe(struct device_queue_manager *dqm)
  60 {
  61         BUG_ON(!dqm);
  62         return dqm->dev->shared_resources.first_compute_pipe;
  63 }
  64
  65 static inline unsigned int get_pipes_num_cpsch(void)
  66 {
  67         return PIPE_PER_ME_CP_SCHEDULING;
  68 }
  69
  70 static inline unsigned int
  71 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
  72 {
  73         uint32_t nybble;
  74
  75         nybble = (pdd->lds_base >> 60) & 0x0E;
  76
  77         return nybble;
  78
  79 }
  80
  81 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
  82 {
  83         unsigned int shared_base;
  84
  85         shared_base = (pdd->lds_base >> 16) & 0xFF;
  86
  87         return shared_base;
  88 }
  89
  90 static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble);
  91 static void init_process_memory(struct device_queue_manager *dqm,
  92                                 struct qcm_process_device *qpd)
  93 {
  94         struct kfd_process_device *pdd;
  95         unsigned int temp;
  96
  97         BUG_ON(!dqm || !qpd);
  98
  99         pdd = qpd_to_pdd(qpd);
 100
 101         /* check if sh_mem_config register already configured */
 102         if (qpd->sh_mem_config == 0) {
 103                 qpd->sh_mem_config =
 104                         ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) |
 105                         DEFAULT_MTYPE(MTYPE_NONCACHED) |
 106                         APE1_MTYPE(MTYPE_NONCACHED);
 107                 qpd->sh_mem_ape1_limit = 0;
 108                 qpd->sh_mem_ape1_base = 0;
 109         }
 110
 111         if (qpd->pqm->process->is_32bit_user_mode) {
 112                 temp = get_sh_mem_bases_32(pdd);
 113                 qpd->sh_mem_bases = SHARED_BASE(temp);
 114                 qpd->sh_mem_config |= PTR32;
 115         } else {
 116                 temp = get_sh_mem_bases_nybble_64(pdd);
 117                 qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
 118         }
 119
 120         pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
 121                 qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
 122 }
 123
 124 static void program_sh_mem_settings(struct device_queue_manager *dqm,
 125                                         struct qcm_process_device *qpd)
 126 {
 127         return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid,
 128                                                 qpd->sh_mem_config,
 129                                                 qpd->sh_mem_ape1_base,
 130                                                 qpd->sh_mem_ape1_limit,
 131                                                 qpd->sh_mem_bases);
 132 }
 133
 134 static int allocate_vmid(struct device_queue_manager *dqm,
 135                         struct qcm_process_device *qpd,
 136                         struct queue *q)
 137 {
 138         int bit, allocated_vmid;
 139
 140         if (dqm->vmid_bitmap == 0)
 141                 return -ENOMEM;
 142
 143         bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
 144         clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
 145
 146         /* Kaveri kfd vmid's starts from vmid 8 */
 147         allocated_vmid = bit + KFD_VMID_START_OFFSET;
 148         pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
 149         qpd->vmid = allocated_vmid;
 150         q->properties.vmid = allocated_vmid;
 151
 152         set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
 153         program_sh_mem_settings(dqm, qpd);
 154
 155         return 0;
 156 }
 157
 158 static void deallocate_vmid(struct device_queue_manager *dqm,
 159                                 struct qcm_process_device *qpd,
 160                                 struct queue *q)
 161 {
 162         int bit = qpd->vmid - KFD_VMID_START_OFFSET;
 163
 164         /* Release the vmid mapping */
 165         set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
 166
 167         set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
 168         qpd->vmid = 0;
 169         q->properties.vmid = 0;
 170 }
 171
 172 static int create_queue_nocpsch(struct device_queue_manager *dqm,
 173                                 struct queue *q,
 174                                 struct qcm_process_device *qpd,
 175                                 int *allocated_vmid)
 176 {
 177         int retval;
 178
 179         BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
 180
 181         pr_debug("kfd: In func %s\n", __func__);
 182         print_queue(q);
 183
 184         mutex_lock(&dqm->lock);
 185
 186         if (list_empty(&qpd->queues_list)) {
 187                 retval = allocate_vmid(dqm, qpd, q);
 188                 if (retval != 0) {
 189                         mutex_unlock(&dqm->lock);
 190                         return retval;
 191                 }
 192         }
 193         *allocated_vmid = qpd->vmid;
 194         q->properties.vmid = qpd->vmid;
 195
 196         retval = create_compute_queue_nocpsch(dqm, q, qpd);
 197
 198         if (retval != 0) {
 199                 if (list_empty(&qpd->queues_list)) {
 200                         deallocate_vmid(dqm, qpd, q);
 201                         *allocated_vmid = 0;
 202                 }
 203                 mutex_unlock(&dqm->lock);
 204                 return retval;
 205         }
 206
 207         list_add(&q->list, &qpd->queues_list);
 208         dqm->queue_count++;
 209
 210         mutex_unlock(&dqm->lock);
 211         return 0;
 212 }
 213
 214 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
 215 {
 216         bool set;
 217         int pipe, bit;
 218
 219         set = false;
 220
 221         for (pipe = dqm->next_pipe_to_allocate; pipe < get_pipes_num(dqm);
 222                         pipe = (pipe + 1) % get_pipes_num(dqm)) {
 223                 if (dqm->allocated_queues[pipe] != 0) {
 224                         bit = find_first_bit(
 225                                 (unsigned long *)&dqm->allocated_queues[pipe],
 226                                 QUEUES_PER_PIPE);
 227
 228                         clear_bit(bit,
 229                                 (unsigned long *)&dqm->allocated_queues[pipe]);
 230                         q->pipe = pipe;
 231                         q->queue = bit;
 232                         set = true;
 233                         break;
 234                 }
 235         }
 236
 237         if (set == false)
 238                 return -EBUSY;
 239
 240         pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
 241                                 __func__, q->pipe, q->queue);
 242         /* horizontal hqd allocation */
 243         dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm);
 244
 245         return 0;
 246 }
 247
 248 static inline void deallocate_hqd(struct device_queue_manager *dqm,
 249                                 struct queue *q)
 250 {
 251         set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
 252 }
 253
 254 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 255                                         struct queue *q,
 256                                         struct qcm_process_device *qpd)
 257 {
 258         int retval;
 259         struct mqd_manager *mqd;
 260
 261         BUG_ON(!dqm || !q || !qpd);
 262
 263         mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
 264         if (mqd == NULL)
 265                 return -ENOMEM;
 266
 267         retval = allocate_hqd(dqm, q);
 268         if (retval != 0)
 269                 return retval;
 270
 271         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 272                                 &q->gart_mqd_addr, &q->properties);
 273         if (retval != 0) {
 274                 deallocate_hqd(dqm, q);
 275                 return retval;
 276         }
 277
 278         pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",
 279                         q->pipe,
 280                         q->queue);
 281
 282         retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
 283                         q->queue, q->properties.write_ptr);
 284         if (retval != 0) {
 285                 deallocate_hqd(dqm, q);
 286                 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 287                 return retval;
 288         }
 289
 290         return 0;
 291 }
 292
 293 static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 294                                 struct qcm_process_device *qpd,
 295                                 struct queue *q)
 296 {
 297         int retval;
 298         struct mqd_manager *mqd;
 299
 300         BUG_ON(!dqm || !q || !q->mqd || !qpd);
 301
 302         retval = 0;
 303
 304         pr_debug("kfd: In Func %s\n", __func__);
 305
 306         mutex_lock(&dqm->lock);
 307         mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
 308         if (mqd == NULL) {
 309                 retval = -ENOMEM;
 310                 goto out;
 311         }
 312
 313         retval = mqd->destroy_mqd(mqd, q->mqd,
 314                                 KFD_PREEMPT_TYPE_WAVEFRONT,
 315                                 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
 316                                 q->pipe, q->queue);
 317
 318         if (retval != 0)
 319                 goto out;
 320
 321         deallocate_hqd(dqm, q);
 322
 323         mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 324
 325         list_del(&q->list);
 326         if (list_empty(&qpd->queues_list))
 327                 deallocate_vmid(dqm, qpd, q);
 328         dqm->queue_count--;
 329 out:
 330         mutex_unlock(&dqm->lock);
 331         return retval;
 332 }
 333
 334 static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 335 {
 336         int retval;
 337         struct mqd_manager *mqd;
 338         bool prev_active = false;
 339
 340         BUG_ON(!dqm || !q || !q->mqd);
 341
 342         mutex_lock(&dqm->lock);
 343         mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
 344         if (mqd == NULL) {
 345                 mutex_unlock(&dqm->lock);
 346                 return -ENOMEM;
 347         }
 348
 349         if (q->properties.is_active == true)
 350                 prev_active = true;
 351
 352         /*
 353          *
 354          * check active state vs. the previous state
 355          * and modify counter accordingly
 356          */
 357         retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
 358         if ((q->properties.is_active == true) && (prev_active == false))
 359                 dqm->queue_count++;
 360         else if ((q->properties.is_active == false) && (prev_active == true))
 361                 dqm->queue_count--;
 362
 363         if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
 364                 retval = execute_queues_cpsch(dqm, false);
 365
 366         mutex_unlock(&dqm->lock);
 367         return retval;
 368 }
 369
 370 static struct mqd_manager *get_mqd_manager_nocpsch(
 371                 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
 372 {
 373         struct mqd_manager *mqd;
 374
 375         BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
 376
 377         pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
 378
 379         mqd = dqm->mqds[type];
 380         if (!mqd) {
 381                 mqd = mqd_manager_init(type, dqm->dev);
 382                 if (mqd == NULL)
 383                         pr_err("kfd: mqd manager is NULL");
 384                 dqm->mqds[type] = mqd;
 385         }
 386
 387         return mqd;
 388 }
 389
 390 static int register_process_nocpsch(struct device_queue_manager *dqm,
 391                                         struct qcm_process_device *qpd)
 392 {
 393         struct device_process_node *n;
 394
 395         BUG_ON(!dqm || !qpd);
 396
 397         pr_debug("kfd: In func %s\n", __func__);
 398
 399         n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
 400         if (!n)
 401                 return -ENOMEM;
 402
 403         n->qpd = qpd;
 404
 405         mutex_lock(&dqm->lock);
 406         list_add(&n->list, &dqm->queues);
 407
 408         init_process_memory(dqm, qpd);
 409         dqm->processes_count++;
 410
 411         mutex_unlock(&dqm->lock);
 412
 413         return 0;
 414 }
 415
 416 static int unregister_process_nocpsch(struct device_queue_manager *dqm,
 417                                         struct qcm_process_device *qpd)
 418 {
 419         int retval;
 420         struct device_process_node *cur, *next;
 421
 422         BUG_ON(!dqm || !qpd);
 423
 424         BUG_ON(!list_empty(&qpd->queues_list));
 425
 426         pr_debug("kfd: In func %s\n", __func__);
 427
 428         retval = 0;
 429         mutex_lock(&dqm->lock);
 430
 431         list_for_each_entry_safe(cur, next, &dqm->queues, list) {
 432                 if (qpd == cur->qpd) {
 433                         list_del(&cur->list);
 434                         kfree(cur);
 435                         dqm->processes_count--;
 436                         goto out;
 437                 }
 438         }
 439         /* qpd not found in dqm list */
 440         retval = 1;
 441 out:
 442         mutex_unlock(&dqm->lock);
 443         return retval;
 444 }
 445
 446 static int
 447 set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
 448                         unsigned int vmid)
 449 {
 450         uint32_t pasid_mapping;
 451
 452         pasid_mapping = (pasid == 0) ? 0 : (uint32_t)pasid |
 453                                                 ATC_VMID_PASID_MAPPING_VALID;
 454         return kfd2kgd->set_pasid_vmid_mapping(dqm->dev->kgd, pasid_mapping,
 455                                                 vmid);
 456 }
 457
 458 static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
 459 {
 460         /* In 64-bit mode, we can only control the top 3 bits of the LDS,
 461          * scratch and GPUVM apertures.
 462          * The hardware fills in the remaining 59 bits according to the
 463          * following pattern:
 464          * LDS:         X0000000'00000000 - X0000001'00000000 (4GB)
 465          * Scratch:     X0000001'00000000 - X0000002'00000000 (4GB)
 466          * GPUVM:       Y0010000'00000000 - Y0020000'00000000 (1TB)
 467          *
 468          * (where X/Y is the configurable nybble with the low-bit 0)
 469          *
 470          * LDS and scratch will have the same top nybble programmed in the
 471          * top 3 bits of SH_MEM_BASES.PRIVATE_BASE.
 472          * GPUVM can have a different top nybble programmed in the
 473          * top 3 bits of SH_MEM_BASES.SHARED_BASE.
 474          * We don't bother to support different top nybbles
 475          * for LDS/Scratch and GPUVM.
 476          */
 477
 478         BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
 479                 top_address_nybble == 0);
 480
 481         return PRIVATE_BASE(top_address_nybble << 12) |
 482                         SHARED_BASE(top_address_nybble << 12);
 483 }
 484
 485 static int init_memory(struct device_queue_manager *dqm)
 486 {
 487         int i, retval;
 488
 489         for (i = 8; i < 16; i++)
 490                 set_pasid_vmid_mapping(dqm, 0, i);
 491
 492         retval = kfd2kgd->init_memory(dqm->dev->kgd);
 493         if (retval == 0)
 494                 is_mem_initialized = true;
 495         return retval;
 496 }
 497
 498
 499 static int init_pipelines(struct device_queue_manager *dqm,
 500                         unsigned int pipes_num, unsigned int first_pipe)
 501 {
 502         void *hpdptr;
 503         struct mqd_manager *mqd;
 504         unsigned int i, err, inx;
 505         uint64_t pipe_hpd_addr;
 506
 507         BUG_ON(!dqm || !dqm->dev);
 508
 509         pr_debug("kfd: In func %s\n", __func__);
 510
 511         /*
 512          * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
 513          * The driver never accesses this memory after zeroing it.
 514          * It doesn't even have to be saved/restored on suspend/resume
 515          * because it contains no data when there are no active queues.
 516          */
 517
 518         err = kfd2kgd->allocate_mem(dqm->dev->kgd,
 519                                 CIK_HPD_EOP_BYTES * pipes_num,
 520                                 PAGE_SIZE,
 521                                 KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
 522                                 (struct kgd_mem **) &dqm->pipeline_mem);
 523
 524         if (err) {
 525                 pr_err("kfd: error allocate vidmem num pipes: %d\n",
 526                         pipes_num);
 527                 return -ENOMEM;
 528         }
 529
 530         hpdptr = dqm->pipeline_mem->cpu_ptr;
 531         dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr;
 532
 533         memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num);
 534
 535         mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_COMPUTE);
 536         if (mqd == NULL) {
 537                 kfd2kgd->free_mem(dqm->dev->kgd,
 538                                 (struct kgd_mem *) dqm->pipeline_mem);
 539                 return -ENOMEM;
 540         }
 541
 542         for (i = 0; i < pipes_num; i++) {
 543                 inx = i + first_pipe;
 544                 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES;
 545                 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr);
 546                 /* = log2(bytes/4)-1 */
 547                 kfd2kgd->init_pipeline(dqm->dev->kgd, i,
 548                                 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr);
 549         }
 550
 551         return 0;
 552 }
 553
 554
 555 static int init_scheduler(struct device_queue_manager *dqm)
 556 {
 557         int retval;
 558
 559         BUG_ON(!dqm);
 560
 561         pr_debug("kfd: In %s\n", __func__);
 562
 563         retval = init_pipelines(dqm, get_pipes_num(dqm), KFD_DQM_FIRST_PIPE);
 564         if (retval != 0)
 565                 return retval;
 566
 567         retval = init_memory(dqm);
 568
 569         return retval;
 570 }
 571
 572 static int initialize_nocpsch(struct device_queue_manager *dqm)
 573 {
 574         int i;
 575
 576         BUG_ON(!dqm);
 577
 578         pr_debug("kfd: In func %s num of pipes: %d\n",
 579                         __func__, get_pipes_num(dqm));
 580
 581         mutex_init(&dqm->lock);
 582         INIT_LIST_HEAD(&dqm->queues);
 583         dqm->queue_count = dqm->next_pipe_to_allocate = 0;
 584         dqm->allocated_queues = kcalloc(get_pipes_num(dqm),
 585                                         sizeof(unsigned int), GFP_KERNEL);
 586         if (!dqm->allocated_queues) {
 587                 mutex_destroy(&dqm->lock);
 588                 return -ENOMEM;
 589         }
 590
 591         for (i = 0; i < get_pipes_num(dqm); i++)
 592                 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1;
 593
 594         dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
 595
 596         init_scheduler(dqm);
 597         return 0;
 598 }
 599
 600 static void uninitialize_nocpsch(struct device_queue_manager *dqm)
 601 {
 602         int i;
 603
 604         BUG_ON(!dqm);
 605
 606         BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
 607
 608         kfree(dqm->allocated_queues);
 609         for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
 610                 kfree(dqm->mqds[i]);
 611         mutex_destroy(&dqm->lock);
 612         kfd2kgd->free_mem(dqm->dev->kgd,
 613                         (struct kgd_mem *) dqm->pipeline_mem);
 614 }
 615
 616 static int start_nocpsch(struct device_queue_manager *dqm)
 617 {
 618         return 0;
 619 }
 620
 621 static int stop_nocpsch(struct device_queue_manager *dqm)
 622 {
 623         return 0;
 624 }
 625
 626 /*
 627  * Device Queue Manager implementation for cp scheduler
 628  */
 629
 630 static int set_sched_resources(struct device_queue_manager *dqm)
 631 {
 632         struct scheduling_resources res;
 633         unsigned int queue_num, queue_mask;
 634
 635         BUG_ON(!dqm);
 636
 637         pr_debug("kfd: In func %s\n", __func__);
 638
 639         queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE;
 640         queue_mask = (1 << queue_num) - 1;
 641         res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
 642         res.vmid_mask <<= KFD_VMID_START_OFFSET;
 643         res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE);
 644         res.gws_mask = res.oac_mask = res.gds_heap_base =
 645                                                 res.gds_heap_size = 0;
 646
 647         pr_debug("kfd: scheduling resources:\n"
 648                         "      vmid mask: 0x%8X\n"
 649                         "      queue mask: 0x%8llX\n",
 650                         res.vmid_mask, res.queue_mask);
 651
 652         return pm_send_set_resources(&dqm->packets, &res);
 653 }
 654
 655 static int initialize_cpsch(struct device_queue_manager *dqm)
 656 {
 657         int retval;
 658
 659         BUG_ON(!dqm);
 660
 661         pr_debug("kfd: In func %s num of pipes: %d\n",
 662                         __func__, get_pipes_num_cpsch());
 663
 664         mutex_init(&dqm->lock);
 665         INIT_LIST_HEAD(&dqm->queues);
 666         dqm->queue_count = dqm->processes_count = 0;
 667         dqm->active_runlist = false;
 668         retval = init_pipelines(dqm, get_pipes_num(dqm), 0);
 669         if (retval != 0)
 670                 goto fail_init_pipelines;
 671
 672         return 0;
 673
 674 fail_init_pipelines:
 675         mutex_destroy(&dqm->lock);
 676         return retval;
 677 }
 678
 679 static int start_cpsch(struct device_queue_manager *dqm)
 680 {
 681         struct device_process_node *node;
 682         int retval;
 683
 684         BUG_ON(!dqm);
 685
 686         retval = 0;
 687
 688         retval = pm_init(&dqm->packets, dqm);
 689         if (retval != 0)
 690                 goto fail_packet_manager_init;
 691
 692         retval = set_sched_resources(dqm);
 693         if (retval != 0)
 694                 goto fail_set_sched_resources;
 695
 696         pr_debug("kfd: allocating fence memory\n");
 697
 698         /* allocate fence memory on the gart */
 699         retval = kfd2kgd->allocate_mem(dqm->dev->kgd,
 700                                         sizeof(*dqm->fence_addr),
 701                                         32,
 702                                         KFD_MEMPOOL_SYSTEM_WRITECOMBINE,
 703                                         (struct kgd_mem **) &dqm->fence_mem);
 704
 705         if (retval != 0)
 706                 goto fail_allocate_vidmem;
 707
 708         dqm->fence_addr = dqm->fence_mem->cpu_ptr;
 709         dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
 710
 711         list_for_each_entry(node, &dqm->queues, list)
 712                 if (node->qpd->pqm->process && dqm->dev)
 713                         kfd_bind_process_to_device(dqm->dev,
 714                                                 node->qpd->pqm->process);
 715
 716         execute_queues_cpsch(dqm, true);
 717
 718         return 0;
 719 fail_allocate_vidmem:
 720 fail_set_sched_resources:
 721         pm_uninit(&dqm->packets);
 722 fail_packet_manager_init:
 723         return retval;
 724 }
 725
 726 static int stop_cpsch(struct device_queue_manager *dqm)
 727 {
 728         struct device_process_node *node;
 729         struct kfd_process_device *pdd;
 730
 731         BUG_ON(!dqm);
 732
 733         destroy_queues_cpsch(dqm, true);
 734
 735         list_for_each_entry(node, &dqm->queues, list) {
 736                 pdd = qpd_to_pdd(node->qpd);
 737                 pdd->bound = false;
 738         }
 739         kfd2kgd->free_mem(dqm->dev->kgd,
 740                         (struct kgd_mem *) dqm->fence_mem);
 741         pm_uninit(&dqm->packets);
 742
 743         return 0;
 744 }
 745
 746 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
 747                                         struct kernel_queue *kq,
 748                                         struct qcm_process_device *qpd)
 749 {
 750         BUG_ON(!dqm || !kq || !qpd);
 751
 752         pr_debug("kfd: In func %s\n", __func__);
 753
 754         mutex_lock(&dqm->lock);
 755         list_add(&kq->list, &qpd->priv_queue_list);
 756         dqm->queue_count++;
 757         qpd->is_debug = true;
 758         execute_queues_cpsch(dqm, false);
 759         mutex_unlock(&dqm->lock);
 760
 761         return 0;
 762 }
 763
 764 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 765                                         struct kernel_queue *kq,
 766                                         struct qcm_process_device *qpd)
 767 {
 768         BUG_ON(!dqm || !kq);
 769
 770         pr_debug("kfd: In %s\n", __func__);
 771
 772         mutex_lock(&dqm->lock);
 773         destroy_queues_cpsch(dqm, false);
 774         list_del(&kq->list);
 775         dqm->queue_count--;
 776         qpd->is_debug = false;
 777         execute_queues_cpsch(dqm, false);
 778         mutex_unlock(&dqm->lock);
 779 }
 780
 781 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 782                         struct qcm_process_device *qpd, int *allocate_vmid)
 783 {
 784         int retval;
 785         struct mqd_manager *mqd;
 786
 787         BUG_ON(!dqm || !q || !qpd);
 788
 789         retval = 0;
 790
 791         if (allocate_vmid)
 792                 *allocate_vmid = 0;
 793
 794         mutex_lock(&dqm->lock);
 795
 796         mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP);
 797         if (mqd == NULL) {
 798                 mutex_unlock(&dqm->lock);
 799                 return -ENOMEM;
 800         }
 801
 802         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 803                                 &q->gart_mqd_addr, &q->properties);
 804         if (retval != 0)
 805                 goto out;
 806
 807         list_add(&q->list, &qpd->queues_list);
 808         if (q->properties.is_active) {
 809                 dqm->queue_count++;
 810                 retval = execute_queues_cpsch(dqm, false);
 811         }
 812
 813 out:
 814         mutex_unlock(&dqm->lock);
 815         return retval;
 816 }
 817
 818 static int fence_wait_timeout(unsigned int *fence_addr,
 819                                 unsigned int fence_value,
 820                                 unsigned long timeout)
 821 {
 822         BUG_ON(!fence_addr);
 823         timeout += jiffies;
 824
 825         while (*fence_addr != fence_value) {
 826                 if (time_after(jiffies, timeout)) {
 827                         pr_err("kfd: qcm fence wait loop timeout expired\n");
 828                         return -ETIME;
 829                 }
 830                 cpu_relax();
 831         }
 832
 833         return 0;
 834 }
 835
 836 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 837 {
 838         int retval;
 839
 840         BUG_ON(!dqm);
 841
 842         retval = 0;
 843
 844         if (lock)
 845                 mutex_lock(&dqm->lock);
 846         if (dqm->active_runlist == false)
 847                 goto out;
 848         retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
 849                         KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0);
 850         if (retval != 0)
 851                 goto out;
 852
 853         *dqm->fence_addr = KFD_FENCE_INIT;
 854         pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
 855                                 KFD_FENCE_COMPLETED);
 856         /* should be timed out */
 857         fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
 858                                 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
 859         pm_release_ib(&dqm->packets);
 860         dqm->active_runlist = false;
 861
 862 out:
 863         if (lock)
 864                 mutex_unlock(&dqm->lock);
 865         return retval;
 866 }
 867
 868 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 869 {
 870         int retval;
 871
 872         BUG_ON(!dqm);
 873
 874         if (lock)
 875                 mutex_lock(&dqm->lock);
 876
 877         retval = destroy_queues_cpsch(dqm, false);
 878         if (retval != 0) {
 879                 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
 880                 goto out;
 881         }
 882
 883         if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
 884                 retval = 0;
 885                 goto out;
 886         }
 887
 888         if (dqm->active_runlist) {
 889                 retval = 0;
 890                 goto out;
 891         }
 892
 893         retval = pm_send_runlist(&dqm->packets, &dqm->queues);
 894         if (retval != 0) {
 895                 pr_err("kfd: failed to execute runlist");
 896                 goto out;
 897         }
 898         dqm->active_runlist = true;
 899
 900 out:
 901         if (lock)
 902                 mutex_unlock(&dqm->lock);
 903         return retval;
 904 }
 905
 906 static int destroy_queue_cpsch(struct device_queue_manager *dqm,
 907                                 struct qcm_process_device *qpd,
 908                                 struct queue *q)
 909 {
 910         int retval;
 911         struct mqd_manager *mqd;
 912
 913         BUG_ON(!dqm || !qpd || !q);
 914
 915         retval = 0;
 916
 917         /* remove queue from list to prevent rescheduling after preemption */
 918         mutex_lock(&dqm->lock);
 919
 920         mqd = dqm->get_mqd_manager(dqm, KFD_MQD_TYPE_CIK_CP);
 921         if (!mqd) {
 922                 retval = -ENOMEM;
 923                 goto failed;
 924         }
 925
 926         list_del(&q->list);
 927         dqm->queue_count--;
 928
 929         execute_queues_cpsch(dqm, false);
 930
 931         mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 932
 933         mutex_unlock(&dqm->lock);
 934
 935         return 0;
 936
 937 failed:
 938         mutex_unlock(&dqm->lock);
 939         return retval;
 940 }
 941
 942 /*
 943  * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
 944  * stay in user mode.
 945  */
 946 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
 947 /* APE1 limit is inclusive and 64K aligned. */
 948 #define APE1_LIMIT_ALIGNMENT 0xFFFF
 949
 950 static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 951                                    struct qcm_process_device *qpd,
 952                                    enum cache_policy default_policy,
 953                                    enum cache_policy alternate_policy,
 954                                    void __user *alternate_aperture_base,
 955                                    uint64_t alternate_aperture_size)
 956 {
 957         uint32_t default_mtype;
 958         uint32_t ape1_mtype;
 959
 960         pr_debug("kfd: In func %s\n", __func__);
 961
 962         mutex_lock(&dqm->lock);
 963
 964         if (alternate_aperture_size == 0) {
 965                 /* base > limit disables APE1 */
 966                 qpd->sh_mem_ape1_base = 1;
 967                 qpd->sh_mem_ape1_limit = 0;
 968         } else {
 969                 /*
 970                  * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
 971                  *                      SH_MEM_APE1_BASE[31:0], 0x0000 }
 972                  * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
 973                  *                      SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
 974                  * Verify that the base and size parameters can be
 975                  * represented in this format and convert them.
 976                  * Additionally restrict APE1 to user-mode addresses.
 977                  */
 978
 979                 uint64_t base = (uintptr_t)alternate_aperture_base;
 980                 uint64_t limit = base + alternate_aperture_size - 1;
 981
 982                 if (limit <= base)
 983                         goto out;
 984
 985                 if ((base & APE1_FIXED_BITS_MASK) != 0)
 986                         goto out;
 987
 988                 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
 989                         goto out;
 990
 991                 qpd->sh_mem_ape1_base = base >> 16;
 992                 qpd->sh_mem_ape1_limit = limit >> 16;
 993         }
 994
 995         default_mtype = (default_policy == cache_policy_coherent) ?
 996                         MTYPE_NONCACHED :
 997                         MTYPE_CACHED;
 998
 999         ape1_mtype = (alternate_policy == cache_policy_coherent) ?
1000                         MTYPE_NONCACHED :
1001                         MTYPE_CACHED;
1002
1003         qpd->sh_mem_config = (qpd->sh_mem_config & PTR32)
1004                         | ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
1005                         | DEFAULT_MTYPE(default_mtype)
1006                         | APE1_MTYPE(ape1_mtype);
1007
1008         if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
1009                 program_sh_mem_settings(dqm, qpd);
1010
1011         pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
1012                 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1013                 qpd->sh_mem_ape1_limit);
1014
1015         mutex_unlock(&dqm->lock);
1016         return true;
1017
1018 out:
1019         mutex_unlock(&dqm->lock);
1020         return false;
1021 }
1022
1023 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1024 {
1025         struct device_queue_manager *dqm;
1026
1027         BUG_ON(!dev);
1028
1029         dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
1030         if (!dqm)
1031                 return NULL;
1032
1033         dqm->dev = dev;
1034         switch (sched_policy) {
1035         case KFD_SCHED_POLICY_HWS:
1036         case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1037                 /* initialize dqm for cp scheduling */
1038                 dqm->create_queue = create_queue_cpsch;
1039                 dqm->initialize = initialize_cpsch;
1040                 dqm->start = start_cpsch;
1041                 dqm->stop = stop_cpsch;
1042                 dqm->destroy_queue = destroy_queue_cpsch;
1043                 dqm->update_queue = update_queue;
1044                 dqm->get_mqd_manager = get_mqd_manager_nocpsch;
1045                 dqm->register_process = register_process_nocpsch;
1046                 dqm->unregister_process = unregister_process_nocpsch;
1047                 dqm->uninitialize = uninitialize_nocpsch;
1048                 dqm->create_kernel_queue = create_kernel_queue_cpsch;
1049                 dqm->destroy_kernel_queue = destroy_kernel_queue_cpsch;
1050                 dqm->set_cache_memory_policy = set_cache_memory_policy;
1051                 break;
1052         case KFD_SCHED_POLICY_NO_HWS:
1053                 /* initialize dqm for no cp scheduling */
1054                 dqm->start = start_nocpsch;
1055                 dqm->stop = stop_nocpsch;
1056                 dqm->create_queue = create_queue_nocpsch;
1057                 dqm->destroy_queue = destroy_queue_nocpsch;
1058                 dqm->update_queue = update_queue;
1059                 dqm->get_mqd_manager = get_mqd_manager_nocpsch;
1060                 dqm->register_process = register_process_nocpsch;
1061                 dqm->unregister_process = unregister_process_nocpsch;
1062                 dqm->initialize = initialize_nocpsch;
1063                 dqm->uninitialize = uninitialize_nocpsch;
1064                 dqm->set_cache_memory_policy = set_cache_memory_policy;
1065                 break;
1066         default:
1067                 BUG();
1068                 break;
1069         }
1070
1071         if (dqm->initialize(dqm) != 0) {
1072                 kfree(dqm);
1073                 return NULL;
1074         }
1075
1076         return dqm;
1077 }
1078
1079 void device_queue_manager_uninit(struct device_queue_manager *dqm)
1080 {
1081         BUG_ON(!dqm);
1082
1083         dqm->uninitialize(dqm);
1084         kfree(dqm);
1085 }
1086