pvr/queue.c

   1 /**********************************************************************
   2  *
   3  * Copyright(c) 2008 Imagination Technologies Ltd. All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful but, except
  10  * as otherwise stated in writing, without any warranty; without even the
  11  * implied warranty of merchantability or fitness for a particular purpose.
  12  * See the GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * Imagination Technologies Ltd. <gpl-support@imgtec.com>
  23  * Home Park Estate, Kings Langley, Herts, WD4 8LZ, UK
  24  *
  25  ******************************************************************************/
  26
  27 #include "services_headers.h"
  28
  29 #include "proc.h"
  30
  31 static int QueuePrintCommands(struct PVRSRV_QUEUE_INFO *psQueue, char *buffer,
  32                               size_t size)
  33 {
  34         off_t off = 0;
  35         int cmds = 0;
  36         u32 ui32ReadOffset = psQueue->ui32ReadOffset;
  37         u32 ui32WriteOffset = psQueue->ui32WriteOffset;
  38         struct PVRSRV_COMMAND *psCmd;
  39
  40         while (ui32ReadOffset != ui32WriteOffset) {
  41                 psCmd = (struct PVRSRV_COMMAND *)((u32) psQueue->pvLinQueueKM +
  42                                         ui32ReadOffset);
  43
  44                 off = printAppend(buffer, size, off,
  45                         "%p %p  %5u  %6u  %3u  %5u   %2u   %2u    %3u  \n",
  46                                 psQueue, psCmd, psCmd->ui32ProcessID,
  47                                 psCmd->CommandType, psCmd->ui32CmdSize,
  48                                 psCmd->ui32DevIndex, psCmd->ui32DstSyncCount,
  49                                 psCmd->ui32SrcSyncCount, psCmd->ui32DataSize);
  50
  51                 ui32ReadOffset += psCmd->ui32CmdSize;
  52                 ui32ReadOffset &= psQueue->ui32QueueSize - 1;
  53                 cmds++;
  54         }
  55         if (cmds == 0)
  56                 off = printAppend(buffer, size, off, "%p <empty>\n", psQueue);
  57         return off;
  58 }
  59
  60 off_t QueuePrintQueues(char *buffer, size_t size, off_t off)
  61 {
  62         struct SYS_DATA *psSysData;
  63         struct PVRSRV_QUEUE_INFO *psQueue;
  64
  65         if (SysAcquireData(&psSysData) != PVRSRV_OK)
  66                 return END_OF_FILE;
  67
  68         if (!off)
  69                 return printAppend(buffer, size, 0,
  70                         "Command Queues\nQueue    CmdPtr      "
  71                            "Pid Command Size DevInd  DSC  SSC  #Data ...\n");
  72
  73         for (psQueue = psSysData->psQueueList; --off && psQueue;
  74              psQueue = psQueue->psNextKM)
  75                 ;
  76
  77         return psQueue ?
  78                 QueuePrintCommands(psQueue, buffer, size) : END_OF_FILE;
  79 }
  80
  81 #define GET_SPACE_IN_CMDQ(psQueue)                                      \
  82         (((psQueue->ui32ReadOffset - psQueue->ui32WriteOffset) +        \
  83           (psQueue->ui32QueueSize - 1)) & (psQueue->ui32QueueSize - 1))
  84
  85 #define UPDATE_QUEUE_WOFF(psQueue, ui32Size)                               \
  86         psQueue->ui32WriteOffset = (psQueue->ui32WriteOffset + ui32Size) & \
  87         (psQueue->ui32QueueSize - 1);
  88
  89 #define SYNCOPS_STALE(ui32OpsComplete, ui32OpsPending)                  \
  90         (ui32OpsComplete >= ui32OpsPending)
  91
  92 static u32 NearestPower2(u32 ui32Value)
  93 {
  94         u32 ui32Temp, ui32Result = 1;
  95
  96         if (!ui32Value)
  97                 return 0;
  98
  99         ui32Temp = ui32Value - 1;
 100         while (ui32Temp) {
 101                 ui32Result <<= 1;
 102                 ui32Temp >>= 1;
 103         }
 104
 105         return ui32Result;
 106 }
 107
 108 enum PVRSRV_ERROR PVRSRVCreateCommandQueueKM(u32 ui32QueueSize,
 109                                  struct PVRSRV_QUEUE_INFO **ppsQueueInfo)
 110 {
 111         struct PVRSRV_QUEUE_INFO *psQueueInfo;
 112         u32 ui32Power2QueueSize = NearestPower2(ui32QueueSize);
 113         struct SYS_DATA *psSysData;
 114         enum PVRSRV_ERROR eError;
 115         void *hMemBlock;
 116
 117         eError = SysAcquireData(&psSysData);
 118         if (eError != PVRSRV_OK)
 119                 return eError;
 120
 121         if (OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 122                        sizeof(struct PVRSRV_QUEUE_INFO),
 123                        (void **) &psQueueInfo, &hMemBlock) != PVRSRV_OK) {
 124                 PVR_DPF(PVR_DBG_ERROR, "PVRSRVCreateCommandQueueKM: "
 125                                         "Failed to alloc queue struct");
 126                 goto ErrorExit;
 127         }
 128         OSMemSet(psQueueInfo, 0, sizeof(struct PVRSRV_QUEUE_INFO));
 129
 130         psQueueInfo->hMemBlock[0] = hMemBlock;
 131         psQueueInfo->ui32ProcessID = OSGetCurrentProcessIDKM();
 132
 133         if (OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 134                        ui32Power2QueueSize + PVRSRV_MAX_CMD_SIZE,
 135                        &psQueueInfo->pvLinQueueKM, &hMemBlock) != PVRSRV_OK) {
 136                 PVR_DPF(PVR_DBG_ERROR, "PVRSRVCreateCommandQueueKM: "
 137                                         "Failed to alloc queue buffer");
 138                 goto ErrorExit;
 139         }
 140
 141         psQueueInfo->hMemBlock[1] = hMemBlock;
 142         psQueueInfo->pvLinQueueUM = psQueueInfo->pvLinQueueKM;
 143
 144         PVR_ASSERT(psQueueInfo->ui32ReadOffset == 0);
 145         PVR_ASSERT(psQueueInfo->ui32WriteOffset == 0);
 146
 147         psQueueInfo->ui32QueueSize = ui32Power2QueueSize;
 148
 149         psQueueInfo->psNextKM = psSysData->psQueueList;
 150         psSysData->psQueueList = psQueueInfo;
 151
 152         *ppsQueueInfo = psQueueInfo;
 153
 154         return PVRSRV_OK;
 155
 156 ErrorExit:
 157
 158         if (psQueueInfo) {
 159                 if (psQueueInfo->pvLinQueueKM)
 160                         OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 161                                   psQueueInfo->ui32QueueSize,
 162                                   psQueueInfo->pvLinQueueKM,
 163                                   psQueueInfo->hMemBlock[1]);
 164
 165                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 166                           sizeof(struct PVRSRV_QUEUE_INFO),
 167                           psQueueInfo, psQueueInfo->hMemBlock[0]);
 168         }
 169
 170         return PVRSRV_ERROR_GENERIC;
 171 }
 172
 173 enum PVRSRV_ERROR PVRSRVDestroyCommandQueueKM(
 174                                 struct PVRSRV_QUEUE_INFO *psQueueInfo)
 175 {
 176         struct PVRSRV_QUEUE_INFO *psQueue;
 177         struct SYS_DATA *psSysData;
 178         enum PVRSRV_ERROR eError;
 179         IMG_BOOL bTimeout = IMG_TRUE;
 180
 181         eError = SysAcquireData(&psSysData);
 182         if (eError != PVRSRV_OK)
 183                 return eError;
 184
 185         psQueue = psSysData->psQueueList;
 186
 187         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US) {
 188                 if (psQueueInfo->ui32ReadOffset ==
 189                     psQueueInfo->ui32WriteOffset) {
 190                         bTimeout = IMG_FALSE;
 191                         break;
 192                 }
 193                 OSWaitus(MAX_HW_TIME_US / WAIT_TRY_COUNT);
 194         }
 195         END_LOOP_UNTIL_TIMEOUT();
 196
 197         if (bTimeout) {
 198                 PVR_DPF(PVR_DBG_ERROR,
 199                          "PVRSRVDestroyCommandQueueKM : Failed to empty queue");
 200                 eError = PVRSRV_ERROR_CANNOT_FLUSH_QUEUE;
 201                 goto ErrorExit;
 202         }
 203
 204         if (psQueue == psQueueInfo) {
 205                 psSysData->psQueueList = psQueueInfo->psNextKM;
 206
 207                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 208                           psQueueInfo->ui32QueueSize,
 209                           psQueueInfo->pvLinQueueKM, psQueueInfo->hMemBlock[1]);
 210                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 211                           sizeof(struct PVRSRV_QUEUE_INFO),
 212                           psQueueInfo, psQueueInfo->hMemBlock[0]);
 213         } else {
 214                 while (psQueue) {
 215                         if (psQueue->psNextKM == psQueueInfo) {
 216                                 psQueue->psNextKM = psQueueInfo->psNextKM;
 217
 218                                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 219                                           psQueueInfo->ui32QueueSize,
 220                                           psQueueInfo->pvLinQueueKM,
 221                                           psQueueInfo->hMemBlock[1]);
 222                                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 223                                           sizeof(struct PVRSRV_QUEUE_INFO),
 224                                           psQueueInfo,
 225                                           psQueueInfo->hMemBlock[0]);
 226                                 break;
 227                         }
 228                         psQueue = psQueue->psNextKM;
 229                 }
 230
 231                 if (!psQueue) {
 232                         eError = PVRSRV_ERROR_INVALID_PARAMS;
 233                         goto ErrorExit;
 234                 }
 235         }
 236
 237 ErrorExit:
 238
 239         return eError;
 240 }
 241
 242 enum PVRSRV_ERROR PVRSRVGetQueueSpaceKM(struct PVRSRV_QUEUE_INFO *psQueue,
 243                                     u32 ui32ParamSize, void **ppvSpace)
 244 {
 245         IMG_BOOL bTimeout = IMG_TRUE;
 246
 247         ui32ParamSize = (ui32ParamSize + 3) & 0xFFFFFFFC;
 248
 249         if (ui32ParamSize > PVRSRV_MAX_CMD_SIZE) {
 250                 PVR_DPF(PVR_DBG_WARNING,
 251                          "PVRSRVGetQueueSpace: max command size is %d bytes",
 252                          PVRSRV_MAX_CMD_SIZE);
 253                 return PVRSRV_ERROR_CMD_TOO_BIG;
 254         }
 255
 256         LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US) {
 257                 if (GET_SPACE_IN_CMDQ(psQueue) > ui32ParamSize) {
 258                         bTimeout = IMG_FALSE;
 259                         break;
 260                 }
 261                 OSWaitus(MAX_HW_TIME_US / WAIT_TRY_COUNT);
 262         }
 263         END_LOOP_UNTIL_TIMEOUT();
 264
 265         if (bTimeout == IMG_TRUE) {
 266                 *ppvSpace = NULL;
 267
 268                 return PVRSRV_ERROR_CANNOT_GET_QUEUE_SPACE;
 269         } else {
 270                 *ppvSpace = (void *)(psQueue->ui32WriteOffset +
 271                                                 (u32)psQueue->pvLinQueueUM);
 272         }
 273
 274         return PVRSRV_OK;
 275 }
 276
 277 enum PVRSRV_ERROR PVRSRVInsertCommandKM(struct PVRSRV_QUEUE_INFO *psQueue,
 278                                    struct PVRSRV_COMMAND **ppsCommand,
 279                                    u32 ui32DevIndex, u16 CommandType,
 280                                    u32 ui32DstSyncCount,
 281                                    struct PVRSRV_KERNEL_SYNC_INFO *apsDstSync[],
 282                                    u32 ui32SrcSyncCount,
 283                                    struct PVRSRV_KERNEL_SYNC_INFO *apsSrcSync[],
 284                                    u32 ui32DataByteSize)
 285 {
 286         enum PVRSRV_ERROR eError;
 287         struct PVRSRV_COMMAND *psCommand;
 288         u32 ui32CommandSize;
 289         u32 i;
 290
 291         ui32DataByteSize = (ui32DataByteSize + 3) & 0xFFFFFFFC;
 292
 293         ui32CommandSize = sizeof(struct PVRSRV_COMMAND) +
 294             ((ui32DstSyncCount + ui32SrcSyncCount) *
 295              sizeof(struct PVRSRV_SYNC_OBJECT)) + ui32DataByteSize;
 296
 297         eError = PVRSRVGetQueueSpaceKM(psQueue, ui32CommandSize,
 298                                   (void **) &psCommand);
 299         if (eError != PVRSRV_OK)
 300                 return eError;
 301
 302         psCommand->ui32ProcessID = OSGetCurrentProcessIDKM();
 303
 304         psCommand->ui32CmdSize = ui32CommandSize;
 305         psCommand->ui32DevIndex = ui32DevIndex;
 306         psCommand->CommandType = CommandType;
 307         psCommand->ui32DstSyncCount = ui32DstSyncCount;
 308         psCommand->ui32SrcSyncCount = ui32SrcSyncCount;
 309         psCommand->psDstSync =
 310             (struct PVRSRV_SYNC_OBJECT *)(((u8 *) psCommand) +
 311                                     sizeof(struct PVRSRV_COMMAND));
 312
 313         psCommand->psSrcSync =
 314             (struct PVRSRV_SYNC_OBJECT *)(((u8 *) psCommand->psDstSync) +
 315                                     (ui32DstSyncCount *
 316                                      sizeof(struct PVRSRV_SYNC_OBJECT)));
 317
 318         psCommand->pvData =
 319             (struct PVRSRV_SYNC_OBJECT *)(((u8 *) psCommand->psSrcSync) +
 320                                     (ui32SrcSyncCount *
 321                                      sizeof(struct PVRSRV_SYNC_OBJECT)));
 322
 323         psCommand->ui32DataSize = ui32DataByteSize;
 324
 325         for (i = 0; i < ui32DstSyncCount; i++) {
 326                 psCommand->psDstSync[i].psKernelSyncInfoKM = apsDstSync[i];
 327                 psCommand->psDstSync[i].ui32WriteOpsPending =
 328                     PVRSRVGetWriteOpsPending(apsDstSync[i], IMG_FALSE);
 329                 psCommand->psDstSync[i].ui32ReadOpsPending =
 330                     PVRSRVGetReadOpsPending(apsDstSync[i], IMG_FALSE);
 331         }
 332
 333         for (i = 0; i < ui32SrcSyncCount; i++) {
 334                 psCommand->psSrcSync[i].psKernelSyncInfoKM = apsSrcSync[i];
 335                 psCommand->psSrcSync[i].ui32WriteOpsPending =
 336                     PVRSRVGetWriteOpsPending(apsSrcSync[i], IMG_TRUE);
 337                 psCommand->psSrcSync[i].ui32ReadOpsPending =
 338                     PVRSRVGetReadOpsPending(apsSrcSync[i], IMG_TRUE);
 339         }
 340
 341         *ppsCommand = psCommand;
 342
 343         return PVRSRV_OK;
 344 }
 345
 346 enum PVRSRV_ERROR PVRSRVSubmitCommandKM(struct PVRSRV_QUEUE_INFO *psQueue,
 347                                     struct PVRSRV_COMMAND *psCommand)
 348 {
 349
 350         if (psCommand->ui32DstSyncCount > 0) {
 351                 psCommand->psDstSync = (struct PVRSRV_SYNC_OBJECT *)
 352                                         (((u8 *)psQueue->pvLinQueueKM) +
 353                                          psQueue->ui32WriteOffset +
 354                                          sizeof(struct PVRSRV_COMMAND));
 355         }
 356
 357         if (psCommand->ui32SrcSyncCount > 0) {
 358                 psCommand->psSrcSync = (struct PVRSRV_SYNC_OBJECT *)
 359                                         (((u8 *)psQueue->pvLinQueueKM) +
 360                                         psQueue->ui32WriteOffset +
 361                                         sizeof(struct PVRSRV_COMMAND) +
 362                                         (psCommand->ui32DstSyncCount *
 363                                            sizeof(struct PVRSRV_SYNC_OBJECT)));
 364         }
 365
 366         psCommand->pvData = (struct PVRSRV_SYNC_OBJECT *)
 367                         (((u8 *)psQueue->pvLinQueueKM) +
 368                          psQueue->ui32WriteOffset +
 369                          sizeof(struct PVRSRV_COMMAND) +
 370                          (psCommand->ui32DstSyncCount *
 371                                 sizeof(struct PVRSRV_SYNC_OBJECT)) +
 372                          (psCommand->ui32SrcSyncCount *
 373                                 sizeof(struct PVRSRV_SYNC_OBJECT)));
 374
 375         UPDATE_QUEUE_WOFF(psQueue, psCommand->ui32CmdSize);
 376
 377         return PVRSRV_OK;
 378 }
 379
 380 static enum PVRSRV_ERROR PVRSRVProcessCommand(struct SYS_DATA *psSysData,
 381                                   struct PVRSRV_COMMAND *psCommand,
 382                                   IMG_BOOL bFlush)
 383 {
 384         struct PVRSRV_SYNC_OBJECT *psWalkerObj;
 385         struct PVRSRV_SYNC_OBJECT *psEndObj;
 386         u32 i;
 387         struct COMMAND_COMPLETE_DATA *psCmdCompleteData;
 388         enum PVRSRV_ERROR eError = PVRSRV_OK;
 389         u32 ui32WriteOpsComplete;
 390         u32 ui32ReadOpsComplete;
 391
 392         psWalkerObj = psCommand->psDstSync;
 393         psEndObj = psWalkerObj + psCommand->ui32DstSyncCount;
 394         while (psWalkerObj < psEndObj) {
 395                 struct PVRSRV_SYNC_DATA *psSyncData =
 396                     psWalkerObj->psKernelSyncInfoKM->psSyncData;
 397
 398                 ui32WriteOpsComplete = psSyncData->ui32WriteOpsComplete;
 399                 ui32ReadOpsComplete = psSyncData->ui32ReadOpsComplete;
 400
 401                 if ((ui32WriteOpsComplete != psWalkerObj->ui32WriteOpsPending)
 402                     || (ui32ReadOpsComplete !=
 403                         psWalkerObj->ui32ReadOpsPending)) {
 404                         if (!bFlush ||
 405                             !SYNCOPS_STALE(ui32WriteOpsComplete,
 406                                            psWalkerObj->ui32WriteOpsPending) ||
 407                             !SYNCOPS_STALE(ui32ReadOpsComplete,
 408                                            psWalkerObj->ui32ReadOpsPending)) {
 409                                 return PVRSRV_ERROR_FAILED_DEPENDENCIES;
 410                         }
 411                 }
 412
 413                 psWalkerObj++;
 414         }
 415
 416         psWalkerObj = psCommand->psSrcSync;
 417         psEndObj = psWalkerObj + psCommand->ui32SrcSyncCount;
 418         while (psWalkerObj < psEndObj) {
 419                 struct PVRSRV_SYNC_DATA *psSyncData =
 420                     psWalkerObj->psKernelSyncInfoKM->psSyncData;
 421
 422                 ui32ReadOpsComplete = psSyncData->ui32ReadOpsComplete;
 423                 ui32WriteOpsComplete = psSyncData->ui32WriteOpsComplete;
 424
 425                 if ((ui32WriteOpsComplete !=
 426                      psWalkerObj->ui32WriteOpsPending) ||
 427                     (ui32ReadOpsComplete !=
 428                         psWalkerObj->ui32ReadOpsPending)) {
 429                         if (!bFlush &&
 430                             SYNCOPS_STALE(ui32WriteOpsComplete,
 431                                           psWalkerObj->ui32WriteOpsPending) &&
 432                             SYNCOPS_STALE(ui32ReadOpsComplete,
 433                                         psWalkerObj->ui32ReadOpsPending)) {
 434                                 PVR_DPF(PVR_DBG_WARNING,
 435                         "PVRSRVProcessCommand: Stale syncops psSyncData:0x%x "
 436                         "ui32WriteOpsComplete:0x%x ui32WriteOpsPending:0x%x",
 437                                          psSyncData, ui32WriteOpsComplete,
 438                                          psWalkerObj->ui32WriteOpsPending);
 439                         }
 440
 441                         if (!bFlush ||
 442                             !SYNCOPS_STALE(ui32WriteOpsComplete,
 443                                            psWalkerObj->ui32WriteOpsPending) ||
 444                             !SYNCOPS_STALE(ui32ReadOpsComplete,
 445                                            psWalkerObj->ui32ReadOpsPending)) {
 446                                 return PVRSRV_ERROR_FAILED_DEPENDENCIES;
 447                         }
 448                 }
 449                 psWalkerObj++;
 450         }
 451
 452         if (psCommand->ui32DevIndex >= SYS_DEVICE_COUNT) {
 453                 PVR_DPF(PVR_DBG_ERROR,
 454                          "PVRSRVProcessCommand: invalid DeviceType 0x%x",
 455                          psCommand->ui32DevIndex);
 456                 return PVRSRV_ERROR_INVALID_PARAMS;
 457         }
 458
 459         psCmdCompleteData =
 460             psSysData->ppsCmdCompleteData[psCommand->ui32DevIndex][psCommand->
 461                                                                    CommandType];
 462         if (psCmdCompleteData->bInUse)
 463
 464                 return PVRSRV_ERROR_FAILED_DEPENDENCIES;
 465
 466         psCmdCompleteData->bInUse = IMG_TRUE;
 467
 468         psCmdCompleteData->ui32DstSyncCount = psCommand->ui32DstSyncCount;
 469         for (i = 0; i < psCommand->ui32DstSyncCount; i++)
 470                 psCmdCompleteData->psDstSync[i] = psCommand->psDstSync[i];
 471
 472         psCmdCompleteData->ui32SrcSyncCount = psCommand->ui32SrcSyncCount;
 473         for (i = 0; i < psCommand->ui32SrcSyncCount; i++)
 474                 psCmdCompleteData->psSrcSync[i] = psCommand->psSrcSync[i];
 475
 476         if (psSysData->ppfnCmdProcList[psCommand->ui32DevIndex]
 477                                       [psCommand->CommandType]((void *)
 478                                            psCmdCompleteData,
 479                                            psCommand->ui32DataSize,
 480                                            psCommand->pvData) == IMG_FALSE) {
 481                 psCmdCompleteData->bInUse = IMG_FALSE;
 482                 eError = PVRSRV_ERROR_CMD_NOT_PROCESSED;
 483         }
 484
 485         return eError;
 486 }
 487
 488 enum PVRSRV_ERROR PVRSRVProcessQueues(IMG_BOOL bFlush)
 489 {
 490         struct PVRSRV_QUEUE_INFO *psQueue;
 491         struct SYS_DATA *psSysData;
 492         struct PVRSRV_COMMAND *psCommand;
 493         struct PVRSRV_DEVICE_NODE *psDeviceNode;
 494         enum PVRSRV_ERROR eError;
 495
 496         eError = SysAcquireData(&psSysData);
 497         if (eError != PVRSRV_OK)
 498                 return eError;
 499
 500         psQueue = psSysData->psQueueList;
 501
 502         if (!psQueue) {
 503                 PVR_DPF(PVR_DBG_MESSAGE,
 504                          "No Queues installed - cannot process commands");
 505         }
 506
 507         if (bFlush)
 508                 PVRSRVSetDCState(DC_STATE_FLUSH_COMMANDS);
 509
 510         while (psQueue) {
 511                 while (psQueue->ui32ReadOffset != psQueue->ui32WriteOffset) {
 512                         psCommand = (struct PVRSRV_COMMAND *)((u32) psQueue->
 513                                         pvLinQueueKM + psQueue->ui32ReadOffset);
 514
 515                         if (PVRSRVProcessCommand(psSysData, psCommand, bFlush)
 516                             == PVRSRV_OK) {
 517                                 UPDATE_QUEUE_ROFF(psQueue,
 518                                                   psCommand->ui32CmdSize)
 519                                 if (bFlush)
 520                                         continue;
 521                         }
 522                         break;
 523                 }
 524                 psQueue = psQueue->psNextKM;
 525         }
 526
 527         if (bFlush)
 528                 PVRSRVSetDCState(DC_STATE_NO_FLUSH_COMMANDS);
 529
 530         psDeviceNode = psSysData->psDeviceNodeList;
 531         while (psDeviceNode != NULL) {
 532                 if (psDeviceNode->bReProcessDeviceCommandComplete &&
 533                     psDeviceNode->pfnDeviceCommandComplete != NULL) {
 534                         (*psDeviceNode->
 535                          pfnDeviceCommandComplete) (psDeviceNode);
 536                 }
 537                 psDeviceNode = psDeviceNode->psNext;
 538         }
 539
 540         return PVRSRV_OK;
 541 }
 542
 543 void PVRSRVCommandCompleteKM(void *hCmdCookie, IMG_BOOL bScheduleMISR)
 544 {
 545         u32 i;
 546         struct COMMAND_COMPLETE_DATA *psCmdCompleteData =
 547             (struct COMMAND_COMPLETE_DATA *)hCmdCookie;
 548         struct SYS_DATA *psSysData;
 549
 550         if (SysAcquireData(&psSysData) != PVRSRV_OK)
 551                 return;
 552
 553         for (i = 0; i < psCmdCompleteData->ui32DstSyncCount; i++) {
 554                 psCmdCompleteData->psDstSync[i].psKernelSyncInfoKM->psSyncData->
 555                     ui32WriteOpsComplete++;
 556         }
 557
 558         for (i = 0; i < psCmdCompleteData->ui32SrcSyncCount; i++) {
 559                 psCmdCompleteData->psSrcSync[i].psKernelSyncInfoKM->psSyncData->
 560                     ui32ReadOpsComplete++;
 561         }
 562
 563         psCmdCompleteData->bInUse = IMG_FALSE;
 564
 565         PVRSRVCommandCompleteCallbacks();
 566
 567         if (bScheduleMISR)
 568                 OSScheduleMISR(psSysData);
 569 }
 570
 571 void PVRSRVCommandCompleteCallbacks(void)
 572 {
 573         struct SYS_DATA *psSysData;
 574         struct PVRSRV_DEVICE_NODE *psDeviceNode;
 575
 576         if (SysAcquireData(&psSysData) != PVRSRV_OK) {
 577                 PVR_DPF(PVR_DBG_ERROR, "PVRSRVCommandCompleteCallbacks: "
 578                                         "SysAcquireData failed");
 579                 return;
 580         }
 581
 582         psDeviceNode = psSysData->psDeviceNodeList;
 583         while (psDeviceNode != NULL) {
 584                 if (psDeviceNode->pfnDeviceCommandComplete != NULL)
 585                         (*psDeviceNode->pfnDeviceCommandComplete)(psDeviceNode);
 586                 psDeviceNode = psDeviceNode->psNext;
 587         }
 588 }
 589
 590 enum PVRSRV_ERROR PVRSRVRegisterCmdProcListKM(u32 ui32DevIndex,
 591                              IMG_BOOL (**ppfnCmdProcList)(void *, u32, void *),
 592                              u32 ui32MaxSyncsPerCmd[][2], u32 ui32CmdCount)
 593 {
 594         struct SYS_DATA *psSysData;
 595         enum PVRSRV_ERROR eError;
 596         u32 i;
 597         u32 ui32AllocSize;
 598         IMG_BOOL (**ppfnCmdProc)(void *, u32, void *);
 599         struct COMMAND_COMPLETE_DATA *psCmdCompleteData;
 600
 601         if (ui32DevIndex >= SYS_DEVICE_COUNT) {
 602                 PVR_DPF(PVR_DBG_ERROR,
 603                          "PVRSRVRegisterCmdProcListKM: invalid DeviceType 0x%x",
 604                          ui32DevIndex);
 605                 return PVRSRV_ERROR_INVALID_PARAMS;
 606         }
 607
 608         eError = SysAcquireData(&psSysData);
 609         if (eError != PVRSRV_OK) {
 610                 PVR_DPF(PVR_DBG_ERROR,
 611                          "PVRSRVRegisterCmdProcListKM: SysAcquireData failed");
 612                 return eError;
 613         }
 614
 615         eError = OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP, ui32CmdCount *
 616                                      sizeof(IMG_BOOL (*)(void *, u32, void *)),
 617                             (void **)&psSysData->ppfnCmdProcList[ui32DevIndex],
 618                             NULL);
 619         if (eError != PVRSRV_OK) {
 620                 PVR_DPF(PVR_DBG_ERROR,
 621                          "PVRSRVRegisterCmdProcListKM: Failed to alloc queue");
 622                 return eError;
 623         }
 624
 625         ppfnCmdProc = psSysData->ppfnCmdProcList[ui32DevIndex];
 626
 627         for (i = 0; i < ui32CmdCount; i++)
 628                 ppfnCmdProc[i] = ppfnCmdProcList[i];
 629
 630         ui32AllocSize = ui32CmdCount * sizeof(struct COMMAND_COMPLETE_DATA *);
 631         eError = OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 632                             ui32AllocSize,
 633                             (void **) &psSysData->
 634                             ppsCmdCompleteData[ui32DevIndex], NULL);
 635         if (eError != PVRSRV_OK) {
 636                 PVR_DPF(PVR_DBG_ERROR,
 637                         "PVRSRVRegisterCmdProcListKM: Failed to alloc CC data");
 638                 goto ErrorExit;
 639         }
 640
 641         /* clear the list to ensure that we don't try to access uninitialised
 642          * pointer in the 'error' execution path */
 643         OSMemSet(psSysData->ppsCmdCompleteData[ui32DevIndex], 0x00,
 644                  ui32AllocSize);
 645
 646         for (i = 0; i < ui32CmdCount; i++) {
 647                 ui32AllocSize = sizeof(struct COMMAND_COMPLETE_DATA)
 648                     + ((ui32MaxSyncsPerCmd[i][0] + ui32MaxSyncsPerCmd[i][1])
 649                        * sizeof(struct PVRSRV_SYNC_OBJECT));
 650
 651                 eError = OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 652                                     ui32AllocSize,
 653                                     (void **)&psSysData->
 654                                             ppsCmdCompleteData[ui32DevIndex][i],
 655                                     NULL);
 656                 if (eError != PVRSRV_OK) {
 657                         PVR_DPF(PVR_DBG_ERROR, "PVRSRVRegisterCmdProcListKM: "
 658                                                 "Failed to alloc cmd %d",
 659                                  i);
 660                         goto ErrorExit;
 661                 }
 662
 663                 OSMemSet(psSysData->ppsCmdCompleteData[ui32DevIndex][i], 0x00,
 664                          ui32AllocSize);
 665
 666                 psCmdCompleteData =
 667                     psSysData->ppsCmdCompleteData[ui32DevIndex][i];
 668
 669                 psCmdCompleteData->psDstSync = (struct PVRSRV_SYNC_OBJECT *)
 670                      (((u32) psCmdCompleteData) +
 671                                         sizeof(struct COMMAND_COMPLETE_DATA));
 672                 psCmdCompleteData->psSrcSync = (struct PVRSRV_SYNC_OBJECT *)
 673                      (((u32) psCmdCompleteData->psDstSync) +
 674                                         (sizeof(struct PVRSRV_SYNC_OBJECT) *
 675                                                  ui32MaxSyncsPerCmd[i][0]));
 676                 psCmdCompleteData->ui32AllocSize = ui32AllocSize;
 677         }
 678
 679         return PVRSRV_OK;
 680
 681 ErrorExit:
 682
 683         if (psSysData->ppsCmdCompleteData[ui32DevIndex] != NULL) {
 684                 for (i = 0; i < ui32CmdCount; i++) {
 685                         if (psSysData->ppsCmdCompleteData[ui32DevIndex][i] !=
 686                             NULL) {
 687                                 psCmdCompleteData =
 688                                     psSysData->
 689                                             ppsCmdCompleteData[ui32DevIndex][i];
 690                                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 691                                           psCmdCompleteData->ui32AllocSize,
 692                                           psCmdCompleteData, NULL);
 693                         }
 694                 }
 695
 696                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 697                           ui32CmdCount * sizeof(struct COMMAND_COMPLETE_DATA *),
 698                           psSysData->ppsCmdCompleteData[ui32DevIndex],
 699                           NULL);
 700         }
 701
 702         if (psSysData->ppfnCmdProcList[ui32DevIndex] != NULL) {
 703                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP, 0,
 704                           psSysData->ppfnCmdProcList[ui32DevIndex], NULL);
 705         }
 706
 707         return eError;
 708 }
 709
 710 enum PVRSRV_ERROR PVRSRVRemoveCmdProcListKM(u32 ui32DevIndex, u32 ui32CmdCount)
 711 {
 712         struct SYS_DATA *psSysData;
 713         enum PVRSRV_ERROR eError;
 714         u32 i;
 715
 716         if (ui32DevIndex >= SYS_DEVICE_COUNT) {
 717                 PVR_DPF(PVR_DBG_ERROR,
 718                          "PVRSRVRemoveCmdProcListKM: invalid DeviceType 0x%x",
 719                          ui32DevIndex);
 720                 return PVRSRV_ERROR_INVALID_PARAMS;
 721         }
 722
 723         eError = SysAcquireData(&psSysData);
 724         if (eError != PVRSRV_OK) {
 725                 PVR_DPF(PVR_DBG_ERROR,
 726                          "PVRSRVRemoveCmdProcListKM: SysAcquireData failed");
 727                 return eError;
 728         }
 729
 730         if (psSysData->ppsCmdCompleteData[ui32DevIndex] == NULL) {
 731                 PVR_DPF(PVR_DBG_ERROR,
 732                          "PVRSRVRemoveCmdProcListKM: Invalid command array");
 733                 return PVRSRV_ERROR_INVALID_PARAMS;
 734         } else {
 735                 for (i = 0; i < ui32CmdCount; i++) {
 736
 737                         if (psSysData->ppsCmdCompleteData[ui32DevIndex][i] !=
 738                             NULL) {
 739                                 struct COMMAND_COMPLETE_DATA *
 740                                         psCmdCompleteData = psSysData->
 741                                         ppsCmdCompleteData[ui32DevIndex][i];
 742                                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 743                                           psCmdCompleteData->ui32AllocSize,
 744                                           psCmdCompleteData, NULL);
 745                         }
 746                 }
 747
 748                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 749                           ui32CmdCount * sizeof(struct COMMAND_COMPLETE_DATA *),
 750                           psSysData->ppsCmdCompleteData[ui32DevIndex], NULL);
 751         }
 752
 753         if (psSysData->ppfnCmdProcList[ui32DevIndex] != NULL) {
 754                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 755                           ui32CmdCount *
 756                                 sizeof(IMG_BOOL (*)(void *, u32, void *)),
 757                           psSysData->ppfnCmdProcList[ui32DevIndex], NULL);
 758         }
 759
 760         return PVRSRV_OK;
 761 }