pvr/sgxinit.c

   1 /**********************************************************************
   2  *
   3  * Copyright(c) 2008 Imagination Technologies Ltd. All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful but, except
  10  * as otherwise stated in writing, without any warranty; without even the
  11  * implied warranty of merchantability or fitness for a particular purpose.
  12  * See the GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * Imagination Technologies Ltd. <gpl-support@imgtec.com>
  23  * Home Park Estate, Kings Langley, Herts, WD4 8LZ, UK
  24  *
  25  ******************************************************************************/
  26
  27 #include <stddef.h>
  28
  29 #include <linux/workqueue.h>
  30 #include <linux/io.h>
  31 #include <linux/slab.h>
  32 #include <linux/sched.h>
  33
  34 #include "sgxdefs.h"
  35 #include "sgxmmu.h"
  36 #include "services_headers.h"
  37 #include "buffer_manager.h"
  38 #include "sgxapi_km.h"
  39 #include "sgxinfo.h"
  40 #include "sgxinfokm.h"
  41 #include "sgxconfig.h"
  42 #include "sysconfig.h"
  43 #include "pvr_bridge_km.h"
  44 #include "sgx_bridge_km.h"
  45 #include "resman.h"
  46
  47 #include "pdump_km.h"
  48 #include "ra.h"
  49 #include "mmu.h"
  50 #include "handle.h"
  51 #include "perproc.h"
  52
  53 #include "sgxutils.h"
  54 #include "pvrversion.h"
  55 #include "sgx_options.h"
  56
  57 static IMG_BOOL SGX_ISRHandler(void *pvData);
  58
  59 static u32 gui32EventStatusServicesByISR;
  60
  61 static enum PVRSRV_ERROR SGXGetBuildInfoKM(struct PVRSRV_SGXDEV_INFO *psDevInfo,
  62                                     struct PVRSRV_DEVICE_NODE *psDeviceNode);
  63
  64 static void SGXCommandComplete(struct PVRSRV_DEVICE_NODE *psDeviceNode)
  65 {
  66         BUG_ON(in_irq());
  67
  68         SGXScheduleProcessQueuesKM(psDeviceNode);
  69 }
  70
  71 static u32 DeinitDevInfo(struct PVRSRV_SGXDEV_INFO *psDevInfo)
  72 {
  73         if (psDevInfo->psKernelCCBInfo != NULL)
  74                 OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP,
  75                           sizeof(struct PVRSRV_SGX_CCB_INFO),
  76                           psDevInfo->psKernelCCBInfo, NULL);
  77
  78         return PVRSRV_OK;
  79 }
  80
  81 static enum PVRSRV_ERROR InitDevInfo(struct PVRSRV_PER_PROCESS_DATA *psPerProc,
  82                                 struct PVRSRV_DEVICE_NODE *psDeviceNode,
  83                                 struct SGX_BRIDGE_INIT_INFO *psInitInfo)
  84 {
  85         struct PVRSRV_SGXDEV_INFO *psDevInfo =
  86             (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
  87         enum PVRSRV_ERROR eError;
  88
  89         struct PVRSRV_SGX_CCB_INFO *psKernelCCBInfo = NULL;
  90
  91         PVR_UNREFERENCED_PARAMETER(psPerProc);
  92         psDevInfo->sScripts = psInitInfo->sScripts;
  93
  94         psDevInfo->psKernelCCBMemInfo =
  95             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelCCBMemInfo;
  96         psDevInfo->psKernelCCB =
  97             (struct PVRSRV_SGX_KERNEL_CCB *)psDevInfo->psKernelCCBMemInfo->
  98                                                     pvLinAddrKM;
  99
 100         psDevInfo->psKernelCCBCtlMemInfo =
 101             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelCCBCtlMemInfo;
 102         psDevInfo->psKernelCCBCtl =
 103             (struct PVRSRV_SGX_CCB_CTL *)psDevInfo->psKernelCCBCtlMemInfo->
 104                                                     pvLinAddrKM;
 105
 106         psDevInfo->psKernelCCBEventKickerMemInfo =
 107             (struct PVRSRV_KERNEL_MEM_INFO *)
 108                             psInitInfo->hKernelCCBEventKickerMemInfo;
 109         psDevInfo->pui32KernelCCBEventKicker =
 110             (u32 *)psDevInfo->psKernelCCBEventKickerMemInfo->pvLinAddrKM;
 111
 112         psDevInfo->psKernelSGXHostCtlMemInfo =
 113             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->
 114                                                     hKernelSGXHostCtlMemInfo;
 115         psDevInfo->psSGXHostCtl = (struct SGXMKIF_HOST_CTL __force __iomem *)
 116                 psDevInfo->psKernelSGXHostCtlMemInfo->pvLinAddrKM;
 117
 118         psDevInfo->psKernelSGXTA3DCtlMemInfo =
 119             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->
 120                                                     hKernelSGXTA3DCtlMemInfo;
 121
 122         psDevInfo->psKernelSGXMiscMemInfo =
 123             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelSGXMiscMemInfo;
 124
 125         psDevInfo->psKernelHWPerfCBMemInfo =
 126             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelHWPerfCBMemInfo;
 127 #ifdef PVRSRV_USSE_EDM_STATUS_DEBUG
 128         psDevInfo->psKernelEDMStatusBufferMemInfo =
 129             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->
 130                                                   hKernelEDMStatusBufferMemInfo;
 131 #endif
 132
 133         eError = OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
 134                             sizeof(struct PVRSRV_SGX_CCB_INFO),
 135                             (void **)&psKernelCCBInfo, NULL);
 136         if (eError != PVRSRV_OK) {
 137                 PVR_DPF(PVR_DBG_ERROR, "InitDevInfo: Failed to alloc memory");
 138                 goto failed_allockernelccb;
 139         }
 140
 141         OSMemSet(psKernelCCBInfo, 0, sizeof(struct PVRSRV_SGX_CCB_INFO));
 142         psKernelCCBInfo->psCCBMemInfo = psDevInfo->psKernelCCBMemInfo;
 143         psKernelCCBInfo->psCCBCtlMemInfo = psDevInfo->psKernelCCBCtlMemInfo;
 144         psKernelCCBInfo->psCommands = psDevInfo->psKernelCCB->asCommands;
 145         psKernelCCBInfo->pui32WriteOffset =
 146                                 &psDevInfo->psKernelCCBCtl->ui32WriteOffset;
 147         psKernelCCBInfo->pui32ReadOffset =
 148                                 &psDevInfo->psKernelCCBCtl->ui32ReadOffset;
 149         psDevInfo->psKernelCCBInfo = psKernelCCBInfo;
 150
 151         psDevInfo->ui32HostKickAddress = psInitInfo->ui32HostKickAddress;
 152
 153         psDevInfo->ui32GetMiscInfoAddress = psInitInfo->ui32GetMiscInfoAddress;
 154
 155         psDevInfo->bForcePTOff = IMG_FALSE;
 156
 157         psDevInfo->ui32CacheControl = psInitInfo->ui32CacheControl;
 158
 159         psDevInfo->ui32EDMTaskReg0 = psInitInfo->ui32EDMTaskReg0;
 160         psDevInfo->ui32EDMTaskReg1 = psInitInfo->ui32EDMTaskReg1;
 161         psDevInfo->ui32ClkGateStatusReg = psInitInfo->ui32ClkGateStatusReg;
 162         psDevInfo->ui32ClkGateStatusMask = psInitInfo->ui32ClkGateStatusMask;
 163
 164         OSMemCopy(&psDevInfo->asSGXDevData, &psInitInfo->asInitDevData,
 165                   sizeof(psDevInfo->asSGXDevData));
 166
 167         return PVRSRV_OK;
 168
 169 failed_allockernelccb:
 170         DeinitDevInfo(psDevInfo);
 171
 172         return eError;
 173 }
 174
 175 static enum PVRSRV_ERROR SGXRunScript(struct PVRSRV_SGXDEV_INFO *psDevInfo,
 176                                  union SGX_INIT_COMMAND *psScript,
 177                                  u32 ui32NumInitCommands)
 178 {
 179         u32 ui32PC;
 180         union SGX_INIT_COMMAND *psComm;
 181
 182         for (ui32PC = 0, psComm = psScript;
 183              ui32PC < ui32NumInitCommands; ui32PC++, psComm++) {
 184                 switch (psComm->eOp) {
 185                 case SGX_INIT_OP_WRITE_HW_REG:
 186                         {
 187                                 OSWriteHWReg(psDevInfo->pvRegsBaseKM,
 188                                              psComm->sWriteHWReg.ui32Offset,
 189                                              psComm->sWriteHWReg.ui32Value);
 190                                 PDUMPREG(psComm->sWriteHWReg.ui32Offset,
 191                                          psComm->sWriteHWReg.ui32Value);
 192                                 break;
 193                         }
 194 #if defined(PDUMP)
 195                 case SGX_INIT_OP_PDUMP_HW_REG:
 196                         {
 197                                 PDUMPREG(psComm->sPDumpHWReg.ui32Offset,
 198                                          psComm->sPDumpHWReg.ui32Value);
 199                                 break;
 200                         }
 201 #endif
 202                 case SGX_INIT_OP_HALT:
 203                         {
 204                                 return PVRSRV_OK;
 205                         }
 206                 case SGX_INIT_OP_ILLEGAL:
 207
 208                 default:
 209                         {
 210                                 PVR_DPF(PVR_DBG_ERROR,
 211                                      "SGXRunScript: PC %d: Illegal command: %d",
 212                                       ui32PC, psComm->eOp);
 213                                 return PVRSRV_ERROR_GENERIC;
 214                         }
 215                 }
 216
 217         }
 218
 219         return PVRSRV_ERROR_GENERIC;
 220 }
 221
 222 enum PVRSRV_ERROR SGXInitialise(struct PVRSRV_SGXDEV_INFO *psDevInfo,
 223                                   IMG_BOOL bHardwareRecovery)
 224 {
 225         enum PVRSRV_ERROR eError;
 226
 227         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 228                               "SGX initialisation script part 1\n");
 229         eError =
 230             SGXRunScript(psDevInfo, psDevInfo->sScripts.asInitCommandsPart1,
 231                          SGX_MAX_INIT_COMMANDS);
 232         if (eError != PVRSRV_OK) {
 233                 PVR_DPF(PVR_DBG_ERROR,
 234                          "SGXInitialise: SGXRunScript (part 1) failed (%d)",
 235                          eError);
 236                 return PVRSRV_ERROR_GENERIC;
 237         }
 238         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 239                               "End of SGX initialisation script part 1\n");
 240
 241         SGXReset(psDevInfo, PDUMP_FLAGS_CONTINUOUS);
 242
 243
 244
 245         *psDevInfo->pui32KernelCCBEventKicker = 0;
 246 #if defined(PDUMP)
 247         PDUMPMEM(NULL, psDevInfo->psKernelCCBEventKickerMemInfo, 0,
 248                  sizeof(*psDevInfo->pui32KernelCCBEventKicker),
 249                  PDUMP_FLAGS_CONTINUOUS,
 250                  MAKEUNIQUETAG(psDevInfo->psKernelCCBEventKickerMemInfo));
 251 #endif
 252
 253         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 254                               "SGX initialisation script part 2\n");
 255         eError =
 256             SGXRunScript(psDevInfo, psDevInfo->sScripts.asInitCommandsPart2,
 257                          SGX_MAX_INIT_COMMANDS);
 258         if (eError != PVRSRV_OK) {
 259                 PVR_DPF(PVR_DBG_ERROR,
 260                          "SGXInitialise: SGXRunScript (part 2) failed (%d)",
 261                          eError);
 262                 return PVRSRV_ERROR_GENERIC;
 263         }
 264         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 265                               "End of SGX initialisation script part 2\n");
 266
 267         SGXStartTimer(psDevInfo, (IMG_BOOL)!bHardwareRecovery);
 268
 269         if (bHardwareRecovery) {
 270                 struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 271                     psDevInfo->psSGXHostCtl;
 272
 273                 if (PollForValueKM(&psSGXHostCtl->ui32InterruptClearFlags, 0,
 274                      PVRSRV_USSE_EDM_INTERRUPT_HWR,
 275                      MAX_HW_TIME_US / WAIT_TRY_COUNT, 1000) != PVRSRV_OK) {
 276                         PVR_DPF(PVR_DBG_ERROR, "SGXInitialise: "
 277                                         "Wait for uKernel HW Recovery failed");
 278                         PVR_DBG_BREAK;
 279                         return PVRSRV_ERROR_RETRY;
 280                 }
 281         }
 282
 283         PVR_ASSERT(psDevInfo->psKernelCCBCtl->ui32ReadOffset ==
 284                    psDevInfo->psKernelCCBCtl->ui32WriteOffset);
 285
 286         return PVRSRV_OK;
 287 }
 288
 289 enum PVRSRV_ERROR SGXDeinitialise(void *hDevCookie)
 290 {
 291         struct PVRSRV_SGXDEV_INFO *psDevInfo = (struct PVRSRV_SGXDEV_INFO *)
 292                                                                    hDevCookie;
 293         enum PVRSRV_ERROR eError;
 294
 295         if (psDevInfo->pvRegsBaseKM == NULL)
 296                 return PVRSRV_OK;
 297
 298         eError = SGXRunScript(psDevInfo, psDevInfo->sScripts.asDeinitCommands,
 299                          SGX_MAX_DEINIT_COMMANDS);
 300         if (eError != PVRSRV_OK) {
 301                 PVR_DPF(PVR_DBG_ERROR,
 302                          "SGXDeinitialise: SGXRunScript failed (%d)", eError);
 303                 return PVRSRV_ERROR_GENERIC;
 304         }
 305
 306         return PVRSRV_OK;
 307 }
 308
 309 static enum PVRSRV_ERROR DevInitSGXPart1(void *pvDeviceNode)
 310 {
 311         struct PVRSRV_SGXDEV_INFO *psDevInfo;
 312         void *hKernelDevMemContext;
 313         struct IMG_DEV_PHYADDR sPDDevPAddr;
 314         u32 i;
 315         struct PVRSRV_DEVICE_NODE *psDeviceNode = (struct PVRSRV_DEVICE_NODE *)
 316                                                                    pvDeviceNode;
 317         struct DEVICE_MEMORY_HEAP_INFO *psDeviceMemoryHeap =
 318             psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeap;
 319         enum PVRSRV_ERROR eError;
 320
 321         PDUMPCOMMENT("SGX Initialisation Part 1");
 322
 323         PDUMPCOMMENT("SGX Core Version Information: %s",
 324                      SGX_CORE_FRIENDLY_NAME);
 325         PDUMPCOMMENT("SGX Core Revision Information: multi rev support");
 326
 327         if (OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 328                        sizeof(struct PVRSRV_SGXDEV_INFO),
 329                        (void **)&psDevInfo, NULL) != PVRSRV_OK) {
 330                 PVR_DPF(PVR_DBG_ERROR,
 331                         "DevInitSGXPart1 : Failed to alloc memory for DevInfo");
 332                 return PVRSRV_ERROR_OUT_OF_MEMORY;
 333         }
 334         OSMemSet(psDevInfo, 0, sizeof(struct PVRSRV_SGXDEV_INFO));
 335
 336         psDevInfo->eDeviceType = DEV_DEVICE_TYPE;
 337         psDevInfo->eDeviceClass = DEV_DEVICE_CLASS;
 338
 339         psDeviceNode->pvDevice = (void *) psDevInfo;
 340
 341         psDevInfo->pvDeviceMemoryHeap = (void *) psDeviceMemoryHeap;
 342
 343         hKernelDevMemContext = BM_CreateContext(psDeviceNode, &sPDDevPAddr,
 344                                                 NULL, NULL);
 345
 346         psDevInfo->sKernelPDDevPAddr = sPDDevPAddr;
 347
 348         for (i = 0; i < psDeviceNode->sDevMemoryInfo.ui32HeapCount; i++) {
 349                 void *hDevMemHeap;
 350
 351                 switch (psDeviceMemoryHeap[i].DevMemHeapType) {
 352                 case DEVICE_MEMORY_HEAP_KERNEL:
 353                 case DEVICE_MEMORY_HEAP_SHARED:
 354                 case DEVICE_MEMORY_HEAP_SHARED_EXPORTED:
 355                         {
 356                                 hDevMemHeap =
 357                                     BM_CreateHeap(hKernelDevMemContext,
 358                                                   &psDeviceMemoryHeap[i]);
 359
 360                                 psDeviceMemoryHeap[i].hDevMemHeap = hDevMemHeap;
 361                                 break;
 362                         }
 363                 }
 364         }
 365
 366         eError = MMU_BIFResetPDAlloc(psDevInfo);
 367         if (eError != PVRSRV_OK) {
 368                 PVR_DPF(PVR_DBG_ERROR,
 369                          "DevInitSGX : Failed to alloc memory for BIF reset");
 370                 return PVRSRV_ERROR_GENERIC;
 371         }
 372
 373         return PVRSRV_OK;
 374 }
 375
 376 enum PVRSRV_ERROR SGXGetInfoForSrvinitKM(void *hDevHandle,
 377                                 struct SGX_BRIDGE_INFO_FOR_SRVINIT *psInitInfo)
 378 {
 379         struct PVRSRV_DEVICE_NODE *psDeviceNode;
 380         struct PVRSRV_SGXDEV_INFO *psDevInfo;
 381         enum PVRSRV_ERROR eError;
 382
 383         PDUMPCOMMENT("SGXGetInfoForSrvinit");
 384
 385         psDeviceNode = (struct PVRSRV_DEVICE_NODE *)hDevHandle;
 386         psDevInfo = (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 387
 388         psInitInfo->sPDDevPAddr = psDevInfo->sKernelPDDevPAddr;
 389
 390         eError =
 391             PVRSRVGetDeviceMemHeapsKM(hDevHandle, &psInitInfo->asHeapInfo[0]);
 392         if (eError != PVRSRV_OK) {
 393                 PVR_DPF(PVR_DBG_ERROR, "SGXGetInfoForSrvinit: "
 394                                 "PVRSRVGetDeviceMemHeapsKM failed (%d)",
 395                          eError);
 396                 return PVRSRV_ERROR_GENERIC;
 397         }
 398
 399         return eError;
 400 }
 401
 402 enum PVRSRV_ERROR DevInitSGXPart2KM(struct PVRSRV_PER_PROCESS_DATA *psPerProc,
 403                                    void *hDevHandle,
 404                                    struct SGX_BRIDGE_INIT_INFO *psInitInfo)
 405 {
 406         struct PVRSRV_DEVICE_NODE *psDeviceNode;
 407         struct PVRSRV_SGXDEV_INFO *psDevInfo;
 408         enum PVRSRV_ERROR eError;
 409         struct SGX_DEVICE_MAP *psSGXDeviceMap;
 410         enum PVR_POWER_STATE eDefaultPowerState;
 411         u32 l;
 412
 413         PDUMPCOMMENT("SGX Initialisation Part 2");
 414
 415         psDeviceNode = (struct PVRSRV_DEVICE_NODE *)hDevHandle;
 416         psDevInfo = (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 417
 418         eError = InitDevInfo(psPerProc, psDeviceNode, psInitInfo);
 419         if (eError != PVRSRV_OK) {
 420                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM: "
 421                                         "Failed to load EDM program");
 422                 goto failed_init_dev_info;
 423         }
 424
 425
 426         eError = SysGetDeviceMemoryMap(PVRSRV_DEVICE_TYPE_SGX,
 427                                        (void **) &psSGXDeviceMap);
 428         if (eError != PVRSRV_OK) {
 429                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM: "
 430                                         "Failed to get device memory map!");
 431                 return PVRSRV_ERROR_INIT_FAILURE;
 432         }
 433
 434         if (psSGXDeviceMap->pvRegsCpuVBase) {
 435                 psDevInfo->pvRegsBaseKM = psSGXDeviceMap->pvRegsCpuVBase;
 436         } else {
 437                 psDevInfo->pvRegsBaseKM =
 438                     OSMapPhysToLin(psSGXDeviceMap->sRegsCpuPBase,
 439                                    psSGXDeviceMap->ui32RegsSize,
 440                                    PVRSRV_HAP_KERNEL_ONLY | PVRSRV_HAP_UNCACHED,
 441                                    NULL);
 442                 if (!psDevInfo->pvRegsBaseKM) {
 443                         PVR_DPF(PVR_DBG_ERROR,
 444                                  "DevInitSGXPart2KM: Failed to map in regs\n");
 445                         return PVRSRV_ERROR_BAD_MAPPING;
 446                 }
 447         }
 448         psDevInfo->ui32RegSize = psSGXDeviceMap->ui32RegsSize;
 449         psDevInfo->sRegsPhysBase = psSGXDeviceMap->sRegsSysPBase;
 450
 451         psDeviceNode->pvISRData = psDeviceNode;
 452
 453         PVR_ASSERT(psDeviceNode->pfnDeviceISR == SGX_ISRHandler);
 454
 455         pvr_dev_lock();
 456         l = readl(&psDevInfo->psSGXHostCtl->ui32PowerStatus);
 457         l |= PVRSRV_USSE_EDM_POWMAN_NO_WORK;
 458         writel(l, &psDevInfo->psSGXHostCtl->ui32PowerStatus);
 459         pvr_dev_unlock();
 460         eDefaultPowerState = PVRSRV_POWER_STATE_D3;
 461
 462         eError = PVRSRVRegisterPowerDevice(psDeviceNode->sDevId.ui32DeviceIndex,
 463                                            SGXPrePowerStateExt,
 464                                            SGXPostPowerStateExt,
 465                                            SGXPreClockSpeedChange,
 466                                            SGXPostClockSpeedChange,
 467                                            (void *) psDeviceNode,
 468                                            PVRSRV_POWER_STATE_D3,
 469                                            eDefaultPowerState);
 470         if (eError != PVRSRV_OK) {
 471                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM: "
 472                                 "failed to register device with power manager");
 473                 return eError;
 474         }
 475
 476         OSMemSet(psDevInfo->psKernelCCB, 0,
 477                  sizeof(struct PVRSRV_SGX_KERNEL_CCB));
 478         OSMemSet(psDevInfo->psKernelCCBCtl, 0,
 479                  sizeof(struct PVRSRV_SGX_CCB_CTL));
 480         OSMemSet(psDevInfo->pui32KernelCCBEventKicker, 0,
 481                  sizeof(*psDevInfo->pui32KernelCCBEventKicker));
 482         PDUMPCOMMENT("Initialise Kernel CCB");
 483         PDUMPMEM(NULL, psDevInfo->psKernelCCBMemInfo, 0,
 484                  sizeof(struct PVRSRV_SGX_KERNEL_CCB), PDUMP_FLAGS_CONTINUOUS,
 485                  MAKEUNIQUETAG(psDevInfo->psKernelCCBMemInfo));
 486         PDUMPCOMMENT("Initialise Kernel CCB Control");
 487         PDUMPMEM(NULL, psDevInfo->psKernelCCBCtlMemInfo, 0,
 488                  sizeof(struct PVRSRV_SGX_CCB_CTL), PDUMP_FLAGS_CONTINUOUS,
 489                  MAKEUNIQUETAG(psDevInfo->psKernelCCBCtlMemInfo));
 490         PDUMPCOMMENT("Initialise Kernel CCB Event Kicker");
 491         PDUMPMEM(NULL, psDevInfo->psKernelCCBEventKickerMemInfo, 0,
 492                  sizeof(*psDevInfo->pui32KernelCCBEventKicker),
 493                  PDUMP_FLAGS_CONTINUOUS,
 494                  MAKEUNIQUETAG(psDevInfo->psKernelCCBEventKickerMemInfo));
 495
 496         psDevInfo->hTimer = SGXOSTimerInit(psDeviceNode);
 497         if (!psDevInfo->hTimer)
 498                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM : "
 499                         "Failed to initialize HW recovery timer");
 500
 501         return PVRSRV_OK;
 502
 503 failed_init_dev_info:
 504         return eError;
 505 }
 506
 507 static enum PVRSRV_ERROR DevDeInitSGX(void *pvDeviceNode)
 508 {
 509         struct PVRSRV_DEVICE_NODE *psDeviceNode =
 510                 (struct PVRSRV_DEVICE_NODE *)pvDeviceNode;
 511         struct PVRSRV_SGXDEV_INFO *psDevInfo =
 512                 (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 513         enum PVRSRV_ERROR eError;
 514         u32 ui32Heap;
 515         struct DEVICE_MEMORY_HEAP_INFO *psDeviceMemoryHeap;
 516         struct SGX_DEVICE_MAP *psSGXDeviceMap;
 517
 518         if (!psDevInfo) {
 519                 PVR_DPF(PVR_DBG_ERROR, "DevDeInitSGX: Null DevInfo");
 520                 return PVRSRV_OK;
 521         }
 522         if (psDevInfo->hTimer) {
 523                 SGXOSTimerCancel(psDevInfo->hTimer);
 524                 SGXOSTimerDeInit(psDevInfo->hTimer);
 525                 psDevInfo->hTimer = NULL;
 526         }
 527
 528         MMU_BIFResetPDFree(psDevInfo);
 529
 530         DeinitDevInfo(psDevInfo);
 531
 532
 533         psDeviceMemoryHeap =
 534             (struct DEVICE_MEMORY_HEAP_INFO *)psDevInfo->pvDeviceMemoryHeap;
 535         for (ui32Heap = 0;
 536              ui32Heap < psDeviceNode->sDevMemoryInfo.ui32HeapCount;
 537              ui32Heap++) {
 538                 switch (psDeviceMemoryHeap[ui32Heap].DevMemHeapType) {
 539                 case DEVICE_MEMORY_HEAP_KERNEL:
 540                 case DEVICE_MEMORY_HEAP_SHARED:
 541                 case DEVICE_MEMORY_HEAP_SHARED_EXPORTED:
 542                         {
 543                                 if (psDeviceMemoryHeap[ui32Heap].hDevMemHeap !=
 544                                     NULL)
 545                                         BM_DestroyHeap(psDeviceMemoryHeap
 546                                                        [ui32Heap].hDevMemHeap);
 547                                 break;
 548                         }
 549                 }
 550         }
 551
 552         if (!pvr_put_ctx(psDeviceNode->sDevMemoryInfo.pBMKernelContext))
 553                 pr_err("%s: kernel context still in use, can't free it",
 554                         __func__);
 555
 556         eError = PVRSRVRemovePowerDevice(
 557                                 ((struct PVRSRV_DEVICE_NODE *)pvDeviceNode)->
 558                                   sDevId.ui32DeviceIndex);
 559         if (eError != PVRSRV_OK)
 560                 return eError;
 561
 562         eError = SysGetDeviceMemoryMap(PVRSRV_DEVICE_TYPE_SGX,
 563                                        (void **)&psSGXDeviceMap);
 564         if (eError != PVRSRV_OK) {
 565                 PVR_DPF(PVR_DBG_ERROR,
 566                          "DevDeInitSGX: Failed to get device memory map!");
 567                 return eError;
 568         }
 569
 570         if (!psSGXDeviceMap->pvRegsCpuVBase)
 571                 if (psDevInfo->pvRegsBaseKM != NULL)
 572                         OSUnMapPhysToLin(psDevInfo->pvRegsBaseKM,
 573                                          psDevInfo->ui32RegSize,
 574                                          PVRSRV_HAP_KERNEL_ONLY |
 575                                                  PVRSRV_HAP_UNCACHED,
 576                                          NULL);
 577
 578         OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 579                   sizeof(struct PVRSRV_SGXDEV_INFO), psDevInfo, NULL);
 580
 581         psDeviceNode->pvDevice = NULL;
 582
 583         if (psDeviceMemoryHeap != NULL)
 584                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 585                           sizeof(struct DEVICE_MEMORY_HEAP_INFO) *
 586                           psDeviceNode->sDevMemoryInfo.ui32HeapCount,
 587                           psDeviceMemoryHeap, NULL);
 588
 589         return PVRSRV_OK;
 590 }
 591
 592 #ifdef PVRSRV_USSE_EDM_STATUS_DEBUG
 593
 594 #define SGXMK_TRACE_BUFFER_SIZE         512
 595
 596 static void dump_edm(struct PVRSRV_SGXDEV_INFO *psDevInfo)
 597 {
 598         u32 *trace_buffer =
 599                 psDevInfo->psKernelEDMStatusBufferMemInfo->pvLinAddrKM;
 600         u32 last_code, write_offset;
 601         int i;
 602
 603         last_code = *trace_buffer;
 604         trace_buffer++;
 605         write_offset = *trace_buffer;
 606
 607         pr_err("Last SGX microkernel status code: 0x%x\n", last_code);
 608
 609         trace_buffer++;
 610         /* Dump the status values */
 611
 612         for (i = 0; i < SGXMK_TRACE_BUFFER_SIZE; i++) {
 613                 u32     *buf;
 614                 buf = trace_buffer + (((write_offset + i) %
 615                                         SGXMK_TRACE_BUFFER_SIZE) * 4);
 616                 pr_err("(MKT%u) %8.8X %8.8X %8.8X %8.8X\n", i,
 617                                 buf[2], buf[3], buf[1], buf[0]);
 618         }
 619 }
 620 #else
 621 static void dump_edm(struct PVRSRV_SGXDEV_INFO *psDevInfo) {}
 622 #endif
 623
 624 static void dump_process_info(struct PVRSRV_DEVICE_NODE *dev)
 625 {
 626         struct PVRSRV_SGXDEV_INFO *dev_info = dev->pvDevice;
 627         u32 page_dir = readl(dev_info->pvRegsBaseKM +
 628                                 EUR_CR_BIF_DIR_LIST_BASE0);
 629         struct BM_CONTEXT *bm_ctx;
 630         struct RESMAN_CONTEXT *res_ctx = NULL;
 631
 632         bm_ctx = bm_find_context(dev->sDevMemoryInfo.pBMContext, page_dir);
 633         if (bm_ctx)
 634                 res_ctx = pvr_get_resman_ctx(bm_ctx);
 635
 636         if (res_ctx) {
 637                 struct task_struct *tsk;
 638                 struct PVRSRV_PER_PROCESS_DATA *proc;
 639                 int pid;
 640
 641                 proc = pvr_get_proc_by_ctx(res_ctx);
 642                 pid = proc->ui32PID;
 643                 rcu_read_lock();
 644                 tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
 645                 pr_err("PID = %d, process name = %s\n", pid, tsk->comm);
 646                 rcu_read_unlock();
 647         }
 648 }
 649
 650 static void dump_sgx_registers(struct PVRSRV_SGXDEV_INFO *psDevInfo)
 651 {
 652         pr_err("EVENT_STATUS =     0x%08X\n"
 653                 "EVENT_STATUS2 =    0x%08X\n"
 654                 "BIF_CTRL =         0x%08X\n"
 655                 "BIF_INT_STAT =     0x%08X\n"
 656                 "BIF_MEM_REQ_STAT = 0x%08X\n"
 657                 "BIF_FAULT  =       0x%08X\n"
 658                 "CLKGATECTL =       0x%08X\n",
 659                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_EVENT_STATUS),
 660                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_EVENT_STATUS2),
 661                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_CTRL),
 662                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_INT_STAT),
 663                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_MEM_REQ_STAT),
 664                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_FAULT),
 665                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_CLKGATECTL));
 666 }
 667
 668 /* Should be called with pvr_lock held */
 669 void HWRecoveryResetSGX(struct PVRSRV_DEVICE_NODE *psDeviceNode)
 670 {
 671         enum PVRSRV_ERROR eError;
 672         struct PVRSRV_SGXDEV_INFO *psDevInfo =
 673             (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 674         struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 675                                         psDevInfo->psSGXHostCtl;
 676         u32 l;
 677         int max_retries = 10;
 678
 679         BUG_ON(!pvr_is_locked());
 680
 681         pvr_dev_lock();
 682
 683         l = readl(&psSGXHostCtl->ui32InterruptClearFlags);
 684         l |= PVRSRV_USSE_EDM_INTERRUPT_HWR;
 685         writel(l, &psSGXHostCtl->ui32InterruptClearFlags);
 686
 687         pr_err("%s: SGX Hardware Recovery triggered\n", __func__);
 688
 689         dump_process_info(psDeviceNode);
 690         dump_sgx_registers(psDevInfo);
 691         dump_edm(psDevInfo);
 692
 693         PDUMPSUSPEND();
 694
 695         do {
 696                 eError = SGXInitialise(psDevInfo, IMG_TRUE);
 697                 if (eError != PVRSRV_ERROR_RETRY)
 698                         break;
 699         } while (max_retries--);
 700
 701         if (eError != PVRSRV_OK) {
 702                 pr_err("%s: recovery failed (%d). Disabling the driver",
 703                         __func__, eError);
 704                 pvr_disable();
 705
 706                 PDUMPRESUME();
 707                 pvr_dev_unlock();
 708
 709                 return;
 710         }
 711
 712         PDUMPRESUME();
 713
 714         SGXScheduleProcessQueues(psDeviceNode);
 715
 716         pvr_dev_unlock();
 717
 718         PVRSRVProcessQueues(IMG_TRUE);
 719 }
 720
 721 static unsigned long sgx_reset_forced;
 722
 723 static void SGXOSTimer(struct work_struct *work)
 724 {
 725         struct timer_work_data *data = container_of(work,
 726                                                     struct timer_work_data,
 727                                                     work.work);
 728         struct PVRSRV_DEVICE_NODE *psDeviceNode = data->psDeviceNode;
 729         struct PVRSRV_SGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
 730         static u32 ui32EDMTasks;
 731         static u32 ui32LockupCounter;
 732         static u32 ui32NumResets;
 733         u32 ui32CurrentEDMTasks;
 734         IMG_BOOL bLockup = IMG_FALSE;
 735         IMG_BOOL bPoweredDown;
 736
 737         pvr_lock();
 738
 739         if (!data->armed || pvr_is_disabled()) {
 740                 pvr_unlock();
 741                 return;
 742         }
 743
 744         psDevInfo->ui32TimeStamp++;
 745
 746 #if defined(NO_HARDWARE)
 747         bPoweredDown = IMG_TRUE;
 748 #else
 749         bPoweredDown = (IMG_BOOL) !SGXIsDevicePowered(psDeviceNode);
 750 #endif
 751
 752         if (bPoweredDown) {
 753                 ui32LockupCounter = 0;
 754         } else {
 755                 pvr_dev_lock();
 756                 ui32CurrentEDMTasks = OSReadHWReg(psDevInfo->pvRegsBaseKM,
 757                                                 psDevInfo->ui32EDMTaskReg0);
 758                 if (psDevInfo->ui32EDMTaskReg1 != 0)
 759                         ui32CurrentEDMTasks ^=
 760                             OSReadHWReg(psDevInfo->pvRegsBaseKM,
 761                                         psDevInfo->ui32EDMTaskReg1);
 762                 if ((ui32CurrentEDMTasks == ui32EDMTasks) &&
 763                     (psDevInfo->ui32NumResets == ui32NumResets)) {
 764                         ui32LockupCounter++;
 765                         if (ui32LockupCounter == 3) {
 766                                 ui32LockupCounter = 0;
 767                                 PVR_DPF(PVR_DBG_ERROR, "SGXOSTimer() "
 768                                         "detected SGX lockup (0x%x tasks)",
 769                                          ui32EDMTasks);
 770
 771                                 bLockup = IMG_TRUE;
 772                         }
 773                 } else {
 774                         ui32LockupCounter = 0;
 775                         ui32EDMTasks = ui32CurrentEDMTasks;
 776                         ui32NumResets = psDevInfo->ui32NumResets;
 777                 }
 778                 pvr_dev_unlock();
 779         }
 780
 781         bLockup |= cmpxchg(&sgx_reset_forced, 1, 0);
 782
 783         if (bLockup) {
 784                 struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 785                                                 psDevInfo->psSGXHostCtl;
 786                 u32 l;
 787
 788                 pvr_dev_lock();
 789                 l = readl(&psSGXHostCtl->ui32HostDetectedLockups);
 790                 l++;
 791                 writel(l, &psSGXHostCtl->ui32HostDetectedLockups);
 792                 pvr_dev_unlock();
 793
 794                 HWRecoveryResetSGX(psDeviceNode);
 795         }
 796
 797         queue_delayed_work(data->work_queue, &data->work,
 798                            msecs_to_jiffies(data->interval));
 799
 800         pvr_unlock();
 801 }
 802
 803 struct timer_work_data *
 804 SGXOSTimerInit(struct PVRSRV_DEVICE_NODE *psDeviceNode)
 805 {
 806         struct timer_work_data *data;
 807
 808         data = kzalloc(sizeof(struct timer_work_data), GFP_KERNEL);
 809         if (!data)
 810                 return NULL;
 811
 812         data->work_queue = create_workqueue("SGXOSTimer");
 813         if (!data->work_queue) {
 814                 kfree(data);
 815                 return NULL;
 816         }
 817
 818         data->interval = 150;
 819         data->psDeviceNode = psDeviceNode;
 820         INIT_DELAYED_WORK(&data->work, SGXOSTimer);
 821
 822         return data;
 823 }
 824
 825 void SGXOSTimerDeInit(struct timer_work_data *data)
 826 {
 827         data->armed = false;
 828         destroy_workqueue(data->work_queue);
 829         kfree(data);
 830 }
 831
 832 enum PVRSRV_ERROR SGXOSTimerEnable(struct timer_work_data *data)
 833 {
 834         if (!data)
 835                 return PVRSRV_ERROR_GENERIC;
 836
 837         if (queue_delayed_work(data->work_queue, &data->work,
 838                                msecs_to_jiffies(data->interval))) {
 839                 data->armed = true;
 840                 return PVRSRV_OK;
 841         }
 842
 843         return PVRSRV_ERROR_GENERIC;
 844 }
 845
 846 enum PVRSRV_ERROR SGXOSTimerCancel(struct timer_work_data *data)
 847 {
 848         if (!data)
 849                 return PVRSRV_ERROR_GENERIC;
 850
 851         data->armed = false;
 852         cancel_delayed_work(&data->work);
 853
 854         return PVRSRV_OK;
 855 }
 856
 857 int sgx_force_reset(void)
 858 {
 859         return !cmpxchg(&sgx_reset_forced, 0, 1);
 860 }
 861
 862 static IMG_BOOL SGX_ISRHandler(void *pvData)
 863 {
 864         IMG_BOOL bInterruptProcessed = IMG_FALSE;
 865
 866         {
 867                 u32 ui32EventStatus, ui32EventEnable;
 868                 u32 ui32EventClear = 0;
 869                 struct PVRSRV_DEVICE_NODE *psDeviceNode;
 870                 struct PVRSRV_SGXDEV_INFO *psDevInfo;
 871
 872                 if (pvData == NULL) {
 873                         PVR_DPF(PVR_DBG_ERROR,
 874                                  "SGX_ISRHandler: Invalid params\n");
 875                         return bInterruptProcessed;
 876                 }
 877
 878                 psDeviceNode = (struct PVRSRV_DEVICE_NODE *)pvData;
 879                 psDevInfo = (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 880
 881                 ui32EventStatus =
 882                     OSReadHWReg(psDevInfo->pvRegsBaseKM, EUR_CR_EVENT_STATUS);
 883                 ui32EventEnable = OSReadHWReg(psDevInfo->pvRegsBaseKM,
 884                                                 EUR_CR_EVENT_HOST_ENABLE);
 885
 886                 gui32EventStatusServicesByISR = ui32EventStatus;
 887
 888                 ui32EventStatus &= ui32EventEnable;
 889
 890                 if (ui32EventStatus & EUR_CR_EVENT_STATUS_SW_EVENT_MASK)
 891                         ui32EventClear |= EUR_CR_EVENT_HOST_CLEAR_SW_EVENT_MASK;
 892
 893                 if (ui32EventClear) {
 894                         bInterruptProcessed = IMG_TRUE;
 895
 896                         ui32EventClear |=
 897                             EUR_CR_EVENT_HOST_CLEAR_MASTER_INTERRUPT_MASK;
 898
 899                         OSWriteHWReg(psDevInfo->pvRegsBaseKM,
 900                                      EUR_CR_EVENT_HOST_CLEAR, ui32EventClear);
 901                 }
 902         }
 903
 904         return bInterruptProcessed;
 905 }
 906
 907 static void SGX_MISRHandler(void *pvData)
 908 {
 909         struct PVRSRV_DEVICE_NODE *psDeviceNode =
 910                         (struct PVRSRV_DEVICE_NODE *)pvData;
 911         struct PVRSRV_SGXDEV_INFO *psDevInfo =
 912                         (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 913         struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 914                         psDevInfo->psSGXHostCtl;
 915         u32 l1, l2;
 916         int dev_idx;
 917         enum PVRSRV_ERROR err;
 918
 919         dev_idx = psDeviceNode->sDevId.ui32DeviceIndex;
 920
 921         pvr_dev_lock();
 922
 923         err = PVRSRVSetDevicePowerStateKM(dev_idx, PVRSRV_POWER_STATE_D0);
 924         BUG_ON(err != PVRSRV_OK);
 925
 926         l1 = readl(&psSGXHostCtl->ui32InterruptFlags);
 927         l2 = readl(&psSGXHostCtl->ui32InterruptClearFlags);
 928
 929         if ((l1 & PVRSRV_USSE_EDM_INTERRUPT_HWR) &&
 930             !(l2 & PVRSRV_USSE_EDM_INTERRUPT_HWR))
 931                 HWRecoveryResetSGX(psDeviceNode);
 932
 933         if (psDeviceNode->bReProcessDeviceCommandComplete)
 934                 SGXScheduleProcessQueues(psDeviceNode);
 935
 936         SGXTestActivePowerEvent(psDeviceNode);
 937
 938         pvr_dev_unlock();
 939 }
 940
 941 enum PVRSRV_ERROR SGXRegisterDevice(struct PVRSRV_DEVICE_NODE *psDeviceNode)
 942 {
 943         struct DEVICE_MEMORY_INFO *psDevMemoryInfo;
 944         struct DEVICE_MEMORY_HEAP_INFO *psDeviceMemoryHeap;
 945
 946         psDeviceNode->sDevId.eDeviceType = DEV_DEVICE_TYPE;
 947         psDeviceNode->sDevId.eDeviceClass = DEV_DEVICE_CLASS;
 948
 949         psDeviceNode->pfnInitDevice = DevInitSGXPart1;
 950         psDeviceNode->pfnDeInitDevice = DevDeInitSGX;
 951
 952         psDeviceNode->pfnInitDeviceCompatCheck = SGXDevInitCompatCheck;
 953
 954         psDeviceNode->pfnMMUInitialise = MMU_Initialise;
 955         psDeviceNode->pfnMMUFinalise = MMU_Finalise;
 956         psDeviceNode->pfnMMUInsertHeap = MMU_InsertHeap;
 957         psDeviceNode->pfnMMUCreate = MMU_Create;
 958         psDeviceNode->pfnMMUDelete = MMU_Delete;
 959         psDeviceNode->pfnMMUAlloc = MMU_Alloc;
 960         psDeviceNode->pfnMMUFree = MMU_Free;
 961         psDeviceNode->pfnMMUMapPages = MMU_MapPages;
 962         psDeviceNode->pfnMMUMapShadow = MMU_MapShadow;
 963         psDeviceNode->pfnMMUUnmapPages = MMU_UnmapPages;
 964         psDeviceNode->pfnMMUMapScatter = MMU_MapScatter;
 965         psDeviceNode->pfnMMUGetPhysPageAddr = MMU_GetPhysPageAddr;
 966         psDeviceNode->pfnMMUGetPDDevPAddr = MMU_GetPDDevPAddr;
 967
 968         psDeviceNode->pfnDeviceISR = SGX_ISRHandler;
 969         psDeviceNode->pfnDeviceMISR = SGX_MISRHandler;
 970
 971         psDeviceNode->pfnDeviceCommandComplete = SGXCommandComplete;
 972
 973         psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
 974
 975         psDevMemoryInfo->ui32AddressSpaceSizeLog2 =
 976             SGX_FEATURE_ADDRESS_SPACE_SIZE;
 977
 978         psDevMemoryInfo->ui32Flags = 0;
 979         psDevMemoryInfo->ui32HeapCount = SGX_MAX_HEAP_ID;
 980         psDevMemoryInfo->ui32SyncHeapID = SGX_SYNCINFO_HEAP_ID;
 981
 982         psDevMemoryInfo->ui32MappingHeapID = SGX_GENERAL_HEAP_ID;
 983
 984         if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
 985                        sizeof(struct DEVICE_MEMORY_HEAP_INFO) *
 986                        psDevMemoryInfo->ui32HeapCount,
 987                        (void **) &psDevMemoryInfo->psDeviceMemoryHeap,
 988                        NULL) != PVRSRV_OK) {
 989                 PVR_DPF(PVR_DBG_ERROR, "SGXRegisterDevice : "
 990                                 "Failed to alloc memory for "
 991                                 "struct DEVICE_MEMORY_HEAP_INFO");
 992                 return PVRSRV_ERROR_OUT_OF_MEMORY;
 993         }
 994         OSMemSet(psDevMemoryInfo->psDeviceMemoryHeap, 0,
 995                  sizeof(struct DEVICE_MEMORY_HEAP_INFO) *
 996                  psDevMemoryInfo->ui32HeapCount);
 997
 998         psDeviceMemoryHeap = psDevMemoryInfo->psDeviceMemoryHeap;
 999
1000         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32HeapID =
1001             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_GENERAL_HEAP_ID);
1002         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].sDevVAddrBase.uiAddr =
1003             SGX_GENERAL_HEAP_BASE;
1004         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32HeapSize =
1005             SGX_GENERAL_HEAP_SIZE;
1006         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32Attribs =
1007             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1008             PVRSRV_HAP_SINGLE_PROCESS;
1009         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].pszName = "General";
1010         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].pszBSName = "General BS";
1011         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].DevMemHeapType =
1012             DEVICE_MEMORY_HEAP_PERCONTEXT;
1013
1014         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32DataPageSize =
1015             SGX_MMU_PAGE_SIZE;
1016
1017         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32HeapID =
1018             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_TADATA_HEAP_ID);
1019         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].sDevVAddrBase.uiAddr =
1020             SGX_TADATA_HEAP_BASE;
1021         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32HeapSize =
1022             SGX_TADATA_HEAP_SIZE;
1023         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32Attribs =
1024             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION
1025             | PVRSRV_HAP_MULTI_PROCESS;
1026         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].pszName = "TA Data";
1027         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].pszBSName = "TA Data BS";
1028         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].DevMemHeapType =
1029             DEVICE_MEMORY_HEAP_PERCONTEXT;
1030
1031         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32DataPageSize =
1032             SGX_MMU_PAGE_SIZE;
1033
1034         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32HeapID =
1035             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_KERNEL_CODE_HEAP_ID);
1036         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].sDevVAddrBase.uiAddr =
1037             SGX_KERNEL_CODE_HEAP_BASE;
1038         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32HeapSize =
1039             SGX_KERNEL_CODE_HEAP_SIZE;
1040         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32Attribs =
1041             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1042             PVRSRV_HAP_MULTI_PROCESS;
1043         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].pszName = "Kernel Code";
1044         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].pszBSName =
1045             "Kernel Code BS";
1046         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].DevMemHeapType =
1047             DEVICE_MEMORY_HEAP_SHARED_EXPORTED;
1048
1049         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32DataPageSize =
1050             SGX_MMU_PAGE_SIZE;
1051
1052         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32HeapID =
1053             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_KERNEL_DATA_HEAP_ID);
1054         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].sDevVAddrBase.uiAddr =
1055             SGX_KERNEL_DATA_HEAP_BASE;
1056         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32HeapSize =
1057             SGX_KERNEL_DATA_HEAP_SIZE;
1058         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32Attribs =
1059             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1060             PVRSRV_HAP_MULTI_PROCESS;
1061         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].pszName = "KernelData";
1062         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].pszBSName = "KernelData BS";
1063         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].DevMemHeapType =
1064             DEVICE_MEMORY_HEAP_SHARED_EXPORTED;
1065
1066         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32DataPageSize =
1067             SGX_MMU_PAGE_SIZE;
1068
1069         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32HeapID =
1070             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_PIXELSHADER_HEAP_ID);
1071         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].sDevVAddrBase.uiAddr =
1072             SGX_PIXELSHADER_HEAP_BASE;
1073         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32HeapSize =
1074             SGX_PIXELSHADER_HEAP_SIZE;
1075         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32Attribs =
1076             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1077             PVRSRV_HAP_SINGLE_PROCESS;
1078         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].pszName = "PixelShaderUSSE";
1079         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].pszBSName =
1080             "PixelShaderUSSE BS";
1081         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].DevMemHeapType =
1082             DEVICE_MEMORY_HEAP_PERCONTEXT;
1083
1084         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32DataPageSize =
1085             SGX_MMU_PAGE_SIZE;
1086
1087         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32HeapID =
1088             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_VERTEXSHADER_HEAP_ID);
1089         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].sDevVAddrBase.uiAddr =
1090             SGX_VERTEXSHADER_HEAP_BASE;
1091         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32HeapSize =
1092             SGX_VERTEXSHADER_HEAP_SIZE;
1093         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32Attribs =
1094             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1095             PVRSRV_HAP_SINGLE_PROCESS;
1096         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].pszName =
1097             "VertexShaderUSSE";
1098         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].pszBSName =
1099             "VertexShaderUSSE BS";
1100         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].DevMemHeapType =
1101             DEVICE_MEMORY_HEAP_PERCONTEXT;
1102
1103         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32DataPageSize =
1104             SGX_MMU_PAGE_SIZE;
1105
1106         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32HeapID =
1107             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_PDSPIXEL_CODEDATA_HEAP_ID);
1108         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].sDevVAddrBase.uiAddr =
1109             SGX_PDSPIXEL_CODEDATA_HEAP_BASE;
1110         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32HeapSize =
1111             SGX_PDSPIXEL_CODEDATA_HEAP_SIZE;
1112         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32Attribs =
1113             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1114             PVRSRV_HAP_SINGLE_PROCESS;
1115         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].pszName =
1116             "PDSPixelCodeData";
1117         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].pszBSName =
1118             "PDSPixelCodeData BS";
1119         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].DevMemHeapType =
1120             DEVICE_MEMORY_HEAP_PERCONTEXT;
1121
1122         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32DataPageSize =
1123             SGX_MMU_PAGE_SIZE;
1124
1125         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32HeapID =
1126             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_PDSVERTEX_CODEDATA_HEAP_ID);
1127         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].sDevVAddrBase.
1128             uiAddr = SGX_PDSVERTEX_CODEDATA_HEAP_BASE;
1129         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32HeapSize =
1130             SGX_PDSVERTEX_CODEDATA_HEAP_SIZE;
1131         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32Attribs =
1132             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1133             PVRSRV_HAP_SINGLE_PROCESS;
1134         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].pszName =
1135             "PDSVertexCodeData";
1136         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].pszBSName =
1137             "PDSVertexCodeData BS";
1138         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].DevMemHeapType =
1139             DEVICE_MEMORY_HEAP_PERCONTEXT;
1140
1141         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32DataPageSize =
1142             SGX_MMU_PAGE_SIZE;
1143
1144         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32HeapID =
1145             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_SYNCINFO_HEAP_ID);
1146         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].sDevVAddrBase.uiAddr =
1147             SGX_SYNCINFO_HEAP_BASE;
1148         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32HeapSize =
1149             SGX_SYNCINFO_HEAP_SIZE;
1150
1151         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32Attribs =
1152             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1153             PVRSRV_HAP_MULTI_PROCESS;
1154         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].pszName = "CacheCoherent";
1155         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].pszBSName = "CacheCoherent BS";
1156
1157         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].DevMemHeapType =
1158             DEVICE_MEMORY_HEAP_SHARED_EXPORTED;
1159
1160         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32DataPageSize =
1161             SGX_MMU_PAGE_SIZE;
1162
1163         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32HeapID =
1164             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_3DPARAMETERS_HEAP_ID);
1165         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].sDevVAddrBase.uiAddr =
1166             SGX_3DPARAMETERS_HEAP_BASE;
1167         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32HeapSize =
1168             SGX_3DPARAMETERS_HEAP_SIZE;
1169         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].pszName = "3DParameters";
1170         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].pszBSName =
1171             "3DParameters BS";
1172         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32Attribs =
1173             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1174             PVRSRV_HAP_SINGLE_PROCESS;
1175         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].DevMemHeapType =
1176             DEVICE_MEMORY_HEAP_PERCONTEXT;
1177
1178         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32DataPageSize =
1179             SGX_MMU_PAGE_SIZE;
1180
1181         return PVRSRV_OK;
1182 }
1183
1184 enum PVRSRV_ERROR SGXGetClientInfoKM(void *hDevCookie,
1185                                          struct SGX_CLIENT_INFO *psClientInfo)
1186 {
1187         struct PVRSRV_SGXDEV_INFO *psDevInfo =
1188             (struct PVRSRV_SGXDEV_INFO *)
1189                         ((struct PVRSRV_DEVICE_NODE *)hDevCookie)->pvDevice;
1190
1191         psDevInfo->ui32ClientRefCount++;
1192 #ifdef PDUMP
1193         if (psDevInfo->ui32ClientRefCount == 1)
1194                 psDevInfo->psKernelCCBInfo->ui32CCBDumpWOff = 0;
1195 #endif
1196         psClientInfo->ui32ProcessID = OSGetCurrentProcessIDKM();
1197
1198         OSMemCopy(&psClientInfo->asDevData, &psDevInfo->asSGXDevData,
1199                   sizeof(psClientInfo->asDevData));
1200
1201         return PVRSRV_OK;
1202 }
1203
1204 enum PVRSRV_ERROR SGXDevInitCompatCheck(struct PVRSRV_DEVICE_NODE *psDeviceNode)
1205 {
1206         struct PVRSRV_SGXDEV_INFO *psDevInfo;
1207         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo;
1208         enum PVRSRV_ERROR eError;
1209 #if !defined(NO_HARDWARE)
1210         u32 ui32BuildOptions, ui32BuildOptionsMismatch;
1211         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1212 #endif
1213
1214         if (psDeviceNode->sDevId.eDeviceType != PVRSRV_DEVICE_TYPE_SGX) {
1215                 PVR_DPF(PVR_DBG_ERROR,
1216                          "SGXDevInitCompatCheck: Device not of type SGX");
1217                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1218                 goto exit;
1219         }
1220         psDevInfo = psDeviceNode->pvDevice;
1221         psMemInfo = psDevInfo->psKernelSGXMiscMemInfo;
1222
1223 #if !defined(NO_HARDWARE)
1224
1225         eError = SGXGetBuildInfoKM(psDevInfo, psDeviceNode);
1226         if (eError != PVRSRV_OK) {
1227                 PVR_DPF(PVR_DBG_ERROR, "SGXDevInitCompatCheck: "
1228                                 "Unable to validate device DDK version");
1229                 goto exit;
1230         }
1231         psSGXFeatures =
1232             &((struct PVRSRV_SGX_MISCINFO_INFO *)(psMemInfo->pvLinAddrKM))->
1233                                                             sSGXFeatures;
1234         if ((psSGXFeatures->ui32DDKVersion !=
1235              ((PVRVERSION_MAJ << 16) | (PVRVERSION_MIN << 8) |
1236               PVRVERSION_BRANCH)) ||
1237              (psSGXFeatures->ui32DDKBuild != PVRVERSION_BUILD)) {
1238                 PVR_DPF(PVR_DBG_ERROR, "SGXDevInitCompatCheck: "
1239                         "Incompatible driver DDK revision (%ld)"
1240                         "/device DDK revision (%ld).",
1241                          PVRVERSION_BUILD, psSGXFeatures->ui32DDKBuild);
1242                 eError = PVRSRV_ERROR_DDK_VERSION_MISMATCH;
1243                 goto exit;
1244         } else {
1245                 PVR_DPF(PVR_DBG_WARNING, "(Success) SGXInit: "
1246                                 "driver DDK (%ld) and device DDK (%ld) match",
1247                          PVRVERSION_BUILD, psSGXFeatures->ui32DDKBuild);
1248         }
1249
1250         ui32BuildOptions = psSGXFeatures->ui32BuildOptions;
1251         if (ui32BuildOptions != (SGX_BUILD_OPTIONS)) {
1252                 ui32BuildOptionsMismatch =
1253                     ui32BuildOptions ^ (SGX_BUILD_OPTIONS);
1254                 if (((SGX_BUILD_OPTIONS) & ui32BuildOptionsMismatch) != 0)
1255                         PVR_DPF(PVR_DBG_ERROR, "SGXInit: "
1256                                 "Mismatch in driver and microkernel build "
1257                                 "options; extra options present in driver: "
1258                                 "(0x%lx)",
1259                                  (SGX_BUILD_OPTIONS) &
1260                                  ui32BuildOptionsMismatch);
1261
1262                 if ((ui32BuildOptions & ui32BuildOptionsMismatch) != 0)
1263                         PVR_DPF(PVR_DBG_ERROR, "SGXInit: "
1264                                 "Mismatch in driver and microkernel build "
1265                                 "options; extra options present in "
1266                                 "microkernel: (0x%lx)",
1267                                  ui32BuildOptions & ui32BuildOptionsMismatch);
1268                 eError = PVRSRV_ERROR_BUILD_MISMATCH;
1269                 goto exit;
1270         } else {
1271                 PVR_DPF(PVR_DBG_WARNING, "(Success) SGXInit: "
1272                                 "Driver and microkernel build options match.");
1273         }
1274
1275 #endif
1276         eError = PVRSRV_OK;
1277 exit:
1278         return eError;
1279 }
1280
1281 static
1282 enum PVRSRV_ERROR SGXGetBuildInfoKM(struct PVRSRV_SGXDEV_INFO *psDevInfo,
1283                                     struct PVRSRV_DEVICE_NODE *psDeviceNode)
1284 {
1285         enum PVRSRV_ERROR eError;
1286         struct SGXMKIF_COMMAND sCommandData;
1287         struct PVRSRV_SGX_MISCINFO_INFO *psSGXMiscInfoInt;
1288         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1289
1290         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo =
1291             psDevInfo->psKernelSGXMiscMemInfo;
1292
1293         if (!psMemInfo->pvLinAddrKM) {
1294                 PVR_DPF(PVR_DBG_ERROR, "SGXGetMiscInfoKM: Invalid address.");
1295                 return PVRSRV_ERROR_INVALID_PARAMS;
1296         }
1297         psSGXMiscInfoInt = psMemInfo->pvLinAddrKM;
1298         psSGXMiscInfoInt->ui32MiscInfoFlags &= ~PVRSRV_USSE_MISCINFO_READY;
1299         psSGXFeatures = &psSGXMiscInfoInt->sSGXFeatures;
1300
1301         OSMemSet(psMemInfo->pvLinAddrKM, 0,
1302                  sizeof(struct PVRSRV_SGX_MISCINFO_INFO));
1303
1304         sCommandData.ui32Data[1] = psMemInfo->sDevVAddr.uiAddr;
1305
1306         OSMemSet(psSGXFeatures, 0, sizeof(*psSGXFeatures));
1307
1308         mb();
1309
1310         eError = SGXScheduleCCBCommandKM(psDeviceNode,
1311                                          SGXMKIF_COMMAND_REQUEST_SGXMISCINFO,
1312                                          &sCommandData, KERNEL_ID, 0);
1313
1314         if (eError != PVRSRV_OK) {
1315                 PVR_DPF(PVR_DBG_ERROR,
1316                          "SGXGetMiscInfoKM: SGXScheduleCCBCommandKM failed.");
1317                 return eError;
1318         }
1319
1320 #if !defined(NO_HARDWARE)
1321         {
1322                 IMG_BOOL bTimeout = IMG_TRUE;
1323
1324                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US) {
1325                         if ((psSGXMiscInfoInt->
1326                              ui32MiscInfoFlags & PVRSRV_USSE_MISCINFO_READY) !=
1327                             0) {
1328                                 bTimeout = IMG_FALSE;
1329                                 break;
1330                         }
1331                 }
1332                 END_LOOP_UNTIL_TIMEOUT();
1333
1334                 if (bTimeout)
1335                         return PVRSRV_ERROR_TIMEOUT;
1336         }
1337 #endif
1338
1339         return PVRSRV_OK;
1340 }
1341
1342 enum PVRSRV_ERROR SGXGetMiscInfoKM(struct PVRSRV_SGXDEV_INFO *psDevInfo,
1343                                        struct SGX_MISC_INFO *psMiscInfo,
1344                                        struct PVRSRV_DEVICE_NODE *psDeviceNode)
1345 {
1346         switch (psMiscInfo->eRequest) {
1347         case SGX_MISC_INFO_REQUEST_CLOCKSPEED:
1348                 {
1349                         psMiscInfo->uData.ui32SGXClockSpeed =
1350                             psDevInfo->ui32CoreClockSpeed;
1351                         return PVRSRV_OK;
1352                 }
1353
1354         case SGX_MISC_INFO_REQUEST_SGXREV:
1355                 {
1356                         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1357                         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo =
1358                             psDevInfo->psKernelSGXMiscMemInfo;
1359
1360                         SGXGetBuildInfoKM(psDevInfo, psDeviceNode);
1361                         psSGXFeatures =
1362                             &((struct PVRSRV_SGX_MISCINFO_INFO *)(psMemInfo->
1363                                                   pvLinAddrKM))->sSGXFeatures;
1364
1365                         psMiscInfo->uData.sSGXFeatures = *psSGXFeatures;
1366
1367                         PVR_DPF(PVR_DBG_MESSAGE, "SGXGetMiscInfoKM: "
1368                                         "Core 0x%lx, sw ID 0x%lx, "
1369                                         "sw Rev 0x%lx\n",
1370                                  psSGXFeatures->ui32CoreRev,
1371                                  psSGXFeatures->ui32CoreIdSW,
1372                                  psSGXFeatures->ui32CoreRevSW);
1373                         PVR_DPF(PVR_DBG_MESSAGE, "SGXGetMiscInfoKM: "
1374                                         "DDK version 0x%lx, DDK build 0x%lx\n",
1375                                  psSGXFeatures->ui32DDKVersion,
1376                                  psSGXFeatures->ui32DDKBuild);
1377
1378                         return PVRSRV_OK;
1379                 }
1380
1381         case SGX_MISC_INFO_REQUEST_DRIVER_SGXREV:
1382                 {
1383                         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo =
1384                             psDevInfo->psKernelSGXMiscMemInfo;
1385                         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1386
1387                         psSGXFeatures = &((struct PVRSRV_SGX_MISCINFO_INFO *)(
1388                                         psMemInfo->pvLinAddrKM))->sSGXFeatures;
1389
1390                         OSMemSet(psMemInfo->pvLinAddrKM, 0,
1391                                  sizeof(struct PVRSRV_SGX_MISCINFO_INFO));
1392
1393                         psSGXFeatures->ui32DDKVersion =
1394                             (PVRVERSION_MAJ << 16) |
1395                             (PVRVERSION_MIN << 8) | PVRVERSION_BRANCH;
1396                         psSGXFeatures->ui32DDKBuild = PVRVERSION_BUILD;
1397
1398                         psMiscInfo->uData.sSGXFeatures = *psSGXFeatures;
1399                         return PVRSRV_OK;
1400                 }
1401
1402         case SGX_MISC_INFO_REQUEST_SET_HWPERF_STATUS:
1403                 {
1404                         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1405                             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1406                         unsigned ui32MatchingFlags;
1407
1408                         if ((psMiscInfo->uData.ui32NewHWPerfStatus &
1409                              ~(PVRSRV_SGX_HWPERF_GRAPHICS_ON |
1410                                PVRSRV_SGX_HWPERF_MK_EXECUTION_ON)) != 0) {
1411                                 return PVRSRV_ERROR_INVALID_PARAMS;
1412                         }
1413
1414                         pvr_dev_lock();
1415                         ui32MatchingFlags = readl(&psDevInfo->
1416                                                  psSGXHostCtl->ui32HWPerfFlags);
1417                         ui32MatchingFlags &=
1418                                 psMiscInfo->uData.ui32NewHWPerfStatus;
1419                         if ((ui32MatchingFlags & PVRSRV_SGX_HWPERF_GRAPHICS_ON)
1420                             == 0UL) {
1421                                 psHWPerfCB->ui32OrdinalGRAPHICS = 0xffffffff;
1422                         }
1423                         if ((ui32MatchingFlags &
1424                              PVRSRV_SGX_HWPERF_MK_EXECUTION_ON) == 0UL) {
1425                                 psHWPerfCB->ui32OrdinalMK_EXECUTION =
1426                                     0xffffffffUL;
1427                         }
1428
1429
1430                         writel(psMiscInfo->uData.ui32NewHWPerfStatus,
1431                                 &psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1432                         pvr_dev_unlock();
1433 #if defined(PDUMP)
1434                         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
1435                                               "SGX ukernel HWPerf status %u\n",
1436                                               readl(&psDevInfo->psSGXHostCtl->
1437                                                               ui32HWPerfFlags));
1438                         PDUMPMEM(NULL, psDevInfo->psKernelSGXHostCtlMemInfo,
1439                                  offsetof(struct SGXMKIF_HOST_CTL,
1440                                           ui32HWPerfFlags),
1441                                  sizeof(psDevInfo->psSGXHostCtl->
1442                                         ui32HWPerfFlags),
1443                                  PDUMP_FLAGS_CONTINUOUS,
1444                                  MAKEUNIQUETAG(psDevInfo->
1445                                                psKernelSGXHostCtlMemInfo));
1446 #endif
1447
1448                         return PVRSRV_OK;
1449                 }
1450         case SGX_MISC_INFO_REQUEST_HWPERF_CB_ON:
1451                 {
1452
1453                         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1454                             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1455                         u32 l;
1456
1457                         psHWPerfCB->ui32OrdinalGRAPHICS = 0xffffffffUL;
1458
1459                         pvr_dev_lock();;
1460                         l = readl(&psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1461                         l |= PVRSRV_SGX_HWPERF_GRAPHICS_ON;
1462                         writel(l, &psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1463                         pvr_dev_unlock();
1464
1465                         return PVRSRV_OK;
1466                 }
1467         case SGX_MISC_INFO_REQUEST_HWPERF_CB_OFF:
1468                 {
1469                         pvr_dev_lock();
1470                         writel(0, &psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1471                         pvr_dev_unlock();
1472
1473                         return PVRSRV_OK;
1474                 }
1475         case SGX_MISC_INFO_REQUEST_HWPERF_RETRIEVE_CB:
1476                 {
1477                         struct SGX_MISC_INFO_HWPERF_RETRIEVE_CB *psRetrieve =
1478                             &psMiscInfo->uData.sRetrieveCB;
1479                         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1480                             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1481                         unsigned i;
1482
1483                         for (i = 0;
1484                              psHWPerfCB->ui32Woff != psHWPerfCB->ui32Roff
1485                              && i < psRetrieve->ui32ArraySize; i++) {
1486                                 struct SGXMKIF_HWPERF_CB_ENTRY *psData =
1487                                     &psHWPerfCB->psHWPerfCBData[psHWPerfCB->
1488                                                                 ui32Roff];
1489
1490                                 psRetrieve->psHWPerfData[i].ui32FrameNo =
1491                                     psData->ui32FrameNo;
1492                                 psRetrieve->psHWPerfData[i].ui32Type =
1493                                     (psData->ui32Type &
1494                                      PVRSRV_SGX_HWPERF_TYPE_OP_MASK);
1495                                 psRetrieve->psHWPerfData[i].ui32StartTime =
1496                                     psData->ui32Time;
1497                                 psRetrieve->psHWPerfData[i].ui32StartTimeWraps =
1498                                     psData->ui32TimeWraps;
1499                                 psRetrieve->psHWPerfData[i].ui32EndTime =
1500                                     psData->ui32Time;
1501                                 psRetrieve->psHWPerfData[i].ui32EndTimeWraps =
1502                                     psData->ui32TimeWraps;
1503                                 psRetrieve->psHWPerfData[i].ui32ClockSpeed =
1504                                     psDevInfo->ui32CoreClockSpeed;
1505                                 psRetrieve->psHWPerfData[i].ui32TimeMax =
1506                                     psDevInfo->ui32uKernelTimerClock;
1507                                 psHWPerfCB->ui32Roff =
1508                                     (psHWPerfCB->ui32Roff + 1) &
1509                                     (SGXMKIF_HWPERF_CB_SIZE - 1);
1510                         }
1511                         psRetrieve->ui32DataCount = i;
1512                         psRetrieve->ui32Time = OSClockus();
1513                         return PVRSRV_OK;
1514                 }
1515         default:
1516                 {
1517                         return PVRSRV_ERROR_INVALID_PARAMS;
1518                 }
1519         }
1520 }
1521
1522 enum PVRSRV_ERROR SGXReadDiffCountersKM(void *hDevHandle, u32 ui32Reg,
1523                                    u32 *pui32Old, IMG_BOOL bNew, u32 ui32New,
1524                                    u32 ui32NewReset, u32 ui32CountersReg,
1525                                    u32 *pui32Time, IMG_BOOL *pbActive,
1526                                    struct PVRSRV_SGXDEV_DIFF_INFO *psDiffs)
1527 {
1528         struct SYS_DATA *psSysData;
1529         struct PVRSRV_POWER_DEV *psPowerDevice;
1530         IMG_BOOL bPowered = IMG_FALSE;
1531         struct PVRSRV_DEVICE_NODE *psDeviceNode = hDevHandle;
1532         struct PVRSRV_SGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
1533
1534         if (bNew)
1535                 psDevInfo->ui32HWGroupRequested = ui32New;
1536         psDevInfo->ui32HWReset |= ui32NewReset;
1537
1538         SysAcquireData(&psSysData);
1539
1540         psPowerDevice = psSysData->psPowerDeviceList;
1541         while (psPowerDevice) {
1542                 if (psPowerDevice->ui32DeviceIndex ==
1543                     psDeviceNode->sDevId.ui32DeviceIndex) {
1544                         bPowered =
1545                             (IMG_BOOL)(psPowerDevice->eCurrentPowerState ==
1546                                         PVRSRV_POWER_STATE_D0);
1547                         break;
1548                 }
1549
1550                 psPowerDevice = psPowerDevice->psNext;
1551         }
1552
1553         *pbActive = bPowered;
1554
1555         pvr_dev_lock();
1556
1557         {
1558                 struct PVRSRV_SGXDEV_DIFF_INFO sNew,
1559                                                *psPrev = &psDevInfo->sDiffInfo;
1560                 u32 i;
1561
1562                 sNew.ui32Time[0] = OSClockus();
1563                 *pui32Time = sNew.ui32Time[0];
1564                 if (sNew.ui32Time[0] != psPrev->ui32Time[0] && bPowered) {
1565
1566                         *pui32Old =
1567                             OSReadHWReg(psDevInfo->pvRegsBaseKM, ui32Reg);
1568
1569                         for (i = 0; i < PVRSRV_SGX_DIFF_NUM_COUNTERS; ++i) {
1570                                 sNew.aui32Counters[i] =
1571                                     OSReadHWReg(psDevInfo->pvRegsBaseKM,
1572                                                 ui32CountersReg + (i * 4));
1573                         }
1574
1575                         if (psDevInfo->ui32HWGroupRequested != *pui32Old) {
1576                                 if (psDevInfo->ui32HWReset != 0) {
1577                                         OSWriteHWReg(psDevInfo->pvRegsBaseKM,
1578                                                      ui32Reg,
1579                                                      psDevInfo->
1580                                                      ui32HWGroupRequested |
1581                                                      psDevInfo->ui32HWReset);
1582                                         psDevInfo->ui32HWReset = 0;
1583                                 }
1584                                 OSWriteHWReg(psDevInfo->pvRegsBaseKM, ui32Reg,
1585                                              psDevInfo->ui32HWGroupRequested);
1586                         }
1587
1588                         sNew.ui32Marker[0] = psDevInfo->ui32KickTACounter;
1589                         sNew.ui32Marker[1] = psDevInfo->ui32KickTARenderCounter;
1590
1591                         sNew.ui32Time[1] = readl(
1592                                 &psDevInfo->psSGXHostCtl->ui32TimeWraps);
1593
1594                         for (i = 0; i < PVRSRV_SGX_DIFF_NUM_COUNTERS; ++i) {
1595                                 psDiffs->aui32Counters[i] =
1596                                     sNew.aui32Counters[i] -
1597                                     psPrev->aui32Counters[i];
1598                         }
1599
1600                         psDiffs->ui32Marker[0] =
1601                             sNew.ui32Marker[0] - psPrev->ui32Marker[0];
1602                         psDiffs->ui32Marker[1] =
1603                             sNew.ui32Marker[1] - psPrev->ui32Marker[1];
1604
1605                         psDiffs->ui32Time[0] =
1606                             sNew.ui32Time[0] - psPrev->ui32Time[0];
1607                         psDiffs->ui32Time[1] =
1608                             sNew.ui32Time[1] - psPrev->ui32Time[1];
1609
1610                         *psPrev = sNew;
1611                 } else {
1612                         for (i = 0; i < PVRSRV_SGX_DIFF_NUM_COUNTERS; ++i)
1613                                 psDiffs->aui32Counters[i] = 0;
1614
1615                         psDiffs->ui32Marker[0] = 0;
1616                         psDiffs->ui32Marker[1] = 0;
1617
1618                         psDiffs->ui32Time[0] = 0;
1619                         psDiffs->ui32Time[1] = 0;
1620                 }
1621         }
1622
1623         SGXTestActivePowerEvent(psDeviceNode);
1624
1625         pvr_dev_unlock();
1626
1627         return PVRSRV_OK;
1628 }
1629
1630 enum PVRSRV_ERROR SGXReadHWPerfCBKM(void *hDevHandle, u32 ui32ArraySize,
1631                         struct PVRSRV_SGX_HWPERF_CB_ENTRY *psClientHWPerfEntry,
1632                         u32 *pui32DataCount, u32 *pui32ClockSpeed,
1633                         u32 *pui32HostTimeStamp)
1634 {
1635         enum PVRSRV_ERROR eError = PVRSRV_OK;
1636         struct PVRSRV_DEVICE_NODE *psDeviceNode = hDevHandle;
1637         struct PVRSRV_SGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
1638         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1639             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1640         unsigned i;
1641
1642         for (i = 0;
1643              psHWPerfCB->ui32Woff != psHWPerfCB->ui32Roff && i < ui32ArraySize;
1644              i++) {
1645                 struct SGXMKIF_HWPERF_CB_ENTRY *psMKPerfEntry =
1646                     &psHWPerfCB->psHWPerfCBData[psHWPerfCB->ui32Roff];
1647
1648                 psClientHWPerfEntry[i].ui32FrameNo = psMKPerfEntry->ui32FrameNo;
1649                 psClientHWPerfEntry[i].ui32Type = psMKPerfEntry->ui32Type;
1650                 psClientHWPerfEntry[i].ui32Ordinal = psMKPerfEntry->ui32Ordinal;
1651                 psClientHWPerfEntry[i].ui32Clocksx16 =
1652                     SGXConvertTimeStamp(psDevInfo, psMKPerfEntry->ui32TimeWraps,
1653                                         psMKPerfEntry->ui32Time);
1654                 OSMemCopy(&psClientHWPerfEntry[i].ui32Counters[0],
1655                           &psMKPerfEntry->ui32Counters[0],
1656                           sizeof(psMKPerfEntry->ui32Counters));
1657
1658                 psHWPerfCB->ui32Roff =
1659                     (psHWPerfCB->ui32Roff + 1) & (SGXMKIF_HWPERF_CB_SIZE - 1);
1660         }
1661
1662         *pui32DataCount = i;
1663         *pui32ClockSpeed = psDevInfo->ui32CoreClockSpeed;
1664         *pui32HostTimeStamp = OSClockus();
1665
1666         return eError;
1667 }