pvr/sgxinit.c

   1 /**********************************************************************
   2  *
   3  * Copyright(c) 2008 Imagination Technologies Ltd. All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful but, except
  10  * as otherwise stated in writing, without any warranty; without even the
  11  * implied warranty of merchantability or fitness for a particular purpose.
  12  * See the GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * Imagination Technologies Ltd. <gpl-support@imgtec.com>
  23  * Home Park Estate, Kings Langley, Herts, WD4 8LZ, UK
  24  *
  25  ******************************************************************************/
  26
  27 #include <stddef.h>
  28
  29 #include <linux/workqueue.h>
  30 #include <linux/io.h>
  31 #include <linux/slab.h>
  32 #include <linux/sched.h>
  33
  34 #include "sgxdefs.h"
  35 #include "sgxmmu.h"
  36 #include "services_headers.h"
  37 #include "buffer_manager.h"
  38 #include "sgxapi_km.h"
  39 #include "sgxinfo.h"
  40 #include "sgxinfokm.h"
  41 #include "sgxconfig.h"
  42 #include "sysconfig.h"
  43 #include "pvr_bridge_km.h"
  44 #include "sgx_bridge_km.h"
  45 #include "resman.h"
  46
  47 #include "pdump_km.h"
  48 #include "ra.h"
  49 #include "mmu.h"
  50 #include "handle.h"
  51 #include "perproc.h"
  52
  53 #include "sgxutils.h"
  54 #include "pvrversion.h"
  55 #include "sgx_options.h"
  56
  57 static IMG_BOOL SGX_ISRHandler(void *pvData);
  58
  59 static u32 gui32EventStatusServicesByISR;
  60
  61 static enum PVRSRV_ERROR SGXGetBuildInfoKM(struct PVRSRV_SGXDEV_INFO *psDevInfo,
  62                                     struct PVRSRV_DEVICE_NODE *psDeviceNode);
  63
  64 static void SGXCommandComplete(struct PVRSRV_DEVICE_NODE *psDeviceNode)
  65 {
  66         BUG_ON(in_irq());
  67
  68         SGXScheduleProcessQueuesKM(psDeviceNode);
  69 }
  70
  71 static u32 DeinitDevInfo(struct PVRSRV_SGXDEV_INFO *psDevInfo)
  72 {
  73         if (psDevInfo->psKernelCCBInfo != NULL)
  74                 OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP,
  75                           sizeof(struct PVRSRV_SGX_CCB_INFO),
  76                           psDevInfo->psKernelCCBInfo, NULL);
  77
  78         return PVRSRV_OK;
  79 }
  80
  81 static enum PVRSRV_ERROR InitDevInfo(struct PVRSRV_PER_PROCESS_DATA *psPerProc,
  82                                 struct PVRSRV_DEVICE_NODE *psDeviceNode,
  83                                 struct SGX_BRIDGE_INIT_INFO *psInitInfo)
  84 {
  85         struct PVRSRV_SGXDEV_INFO *psDevInfo =
  86             (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
  87         enum PVRSRV_ERROR eError;
  88
  89         struct PVRSRV_SGX_CCB_INFO *psKernelCCBInfo = NULL;
  90
  91         PVR_UNREFERENCED_PARAMETER(psPerProc);
  92         psDevInfo->sScripts = psInitInfo->sScripts;
  93
  94         psDevInfo->psKernelCCBMemInfo =
  95             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelCCBMemInfo;
  96         psDevInfo->psKernelCCB =
  97             (struct PVRSRV_SGX_KERNEL_CCB *)psDevInfo->psKernelCCBMemInfo->
  98                                                     pvLinAddrKM;
  99
 100         psDevInfo->psKernelCCBCtlMemInfo =
 101             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelCCBCtlMemInfo;
 102         psDevInfo->psKernelCCBCtl =
 103             (struct PVRSRV_SGX_CCB_CTL *)psDevInfo->psKernelCCBCtlMemInfo->
 104                                                     pvLinAddrKM;
 105
 106         psDevInfo->psKernelCCBEventKickerMemInfo =
 107             (struct PVRSRV_KERNEL_MEM_INFO *)
 108                             psInitInfo->hKernelCCBEventKickerMemInfo;
 109         psDevInfo->pui32KernelCCBEventKicker =
 110             (u32 *)psDevInfo->psKernelCCBEventKickerMemInfo->pvLinAddrKM;
 111
 112         psDevInfo->psKernelSGXHostCtlMemInfo =
 113             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->
 114                                                     hKernelSGXHostCtlMemInfo;
 115         psDevInfo->psSGXHostCtl = (struct SGXMKIF_HOST_CTL __force __iomem *)
 116                 psDevInfo->psKernelSGXHostCtlMemInfo->pvLinAddrKM;
 117
 118         psDevInfo->psKernelSGXTA3DCtlMemInfo =
 119             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->
 120                                                     hKernelSGXTA3DCtlMemInfo;
 121
 122         psDevInfo->psKernelSGXMiscMemInfo =
 123             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelSGXMiscMemInfo;
 124
 125         psDevInfo->psKernelHWPerfCBMemInfo =
 126             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelHWPerfCBMemInfo;
 127         psDevInfo->psKernelEDMStatusBufferMemInfo =
 128                                     psInitInfo->hKernelEDMStatusBufferMemInfo;
 129
 130         eError = OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
 131                             sizeof(struct PVRSRV_SGX_CCB_INFO),
 132                             (void **)&psKernelCCBInfo, NULL);
 133         if (eError != PVRSRV_OK) {
 134                 PVR_DPF(PVR_DBG_ERROR, "InitDevInfo: Failed to alloc memory");
 135                 goto failed_allockernelccb;
 136         }
 137
 138         OSMemSet(psKernelCCBInfo, 0, sizeof(struct PVRSRV_SGX_CCB_INFO));
 139         psKernelCCBInfo->psCCBMemInfo = psDevInfo->psKernelCCBMemInfo;
 140         psKernelCCBInfo->psCCBCtlMemInfo = psDevInfo->psKernelCCBCtlMemInfo;
 141         psKernelCCBInfo->psCommands = psDevInfo->psKernelCCB->asCommands;
 142         psKernelCCBInfo->pui32WriteOffset =
 143                                 &psDevInfo->psKernelCCBCtl->ui32WriteOffset;
 144         psKernelCCBInfo->pui32ReadOffset =
 145                                 &psDevInfo->psKernelCCBCtl->ui32ReadOffset;
 146         psDevInfo->psKernelCCBInfo = psKernelCCBInfo;
 147
 148         psDevInfo->ui32HostKickAddress = psInitInfo->ui32HostKickAddress;
 149
 150         psDevInfo->ui32GetMiscInfoAddress = psInitInfo->ui32GetMiscInfoAddress;
 151
 152         psDevInfo->bForcePTOff = IMG_FALSE;
 153
 154         psDevInfo->ui32CacheControl = psInitInfo->ui32CacheControl;
 155
 156         psDevInfo->ui32EDMTaskReg0 = psInitInfo->ui32EDMTaskReg0;
 157         psDevInfo->ui32EDMTaskReg1 = psInitInfo->ui32EDMTaskReg1;
 158         psDevInfo->ui32ClkGateStatusReg = psInitInfo->ui32ClkGateStatusReg;
 159         psDevInfo->ui32ClkGateStatusMask = psInitInfo->ui32ClkGateStatusMask;
 160
 161         OSMemCopy(&psDevInfo->asSGXDevData, &psInitInfo->asInitDevData,
 162                   sizeof(psDevInfo->asSGXDevData));
 163
 164         return PVRSRV_OK;
 165
 166 failed_allockernelccb:
 167         DeinitDevInfo(psDevInfo);
 168
 169         return eError;
 170 }
 171
 172 static enum PVRSRV_ERROR SGXRunScript(struct PVRSRV_SGXDEV_INFO *psDevInfo,
 173                                  union SGX_INIT_COMMAND *psScript,
 174                                  u32 ui32NumInitCommands)
 175 {
 176         u32 ui32PC;
 177         union SGX_INIT_COMMAND *psComm;
 178
 179         for (ui32PC = 0, psComm = psScript;
 180              ui32PC < ui32NumInitCommands; ui32PC++, psComm++) {
 181                 switch (psComm->eOp) {
 182                 case SGX_INIT_OP_WRITE_HW_REG:
 183                         {
 184                                 OSWriteHWReg(psDevInfo->pvRegsBaseKM,
 185                                              psComm->sWriteHWReg.ui32Offset,
 186                                              psComm->sWriteHWReg.ui32Value);
 187                                 PDUMPREG(psComm->sWriteHWReg.ui32Offset,
 188                                          psComm->sWriteHWReg.ui32Value);
 189                                 break;
 190                         }
 191 #if defined(PDUMP)
 192                 case SGX_INIT_OP_PDUMP_HW_REG:
 193                         {
 194                                 PDUMPREG(psComm->sPDumpHWReg.ui32Offset,
 195                                          psComm->sPDumpHWReg.ui32Value);
 196                                 break;
 197                         }
 198 #endif
 199                 case SGX_INIT_OP_HALT:
 200                         {
 201                                 return PVRSRV_OK;
 202                         }
 203                 case SGX_INIT_OP_ILLEGAL:
 204
 205                 default:
 206                         {
 207                                 PVR_DPF(PVR_DBG_ERROR,
 208                                      "SGXRunScript: PC %d: Illegal command: %d",
 209                                       ui32PC, psComm->eOp);
 210                                 return PVRSRV_ERROR_GENERIC;
 211                         }
 212                 }
 213
 214         }
 215
 216         return PVRSRV_ERROR_GENERIC;
 217 }
 218
 219 enum PVRSRV_ERROR SGXInitialise(struct PVRSRV_SGXDEV_INFO *psDevInfo,
 220                                   IMG_BOOL bHardwareRecovery)
 221 {
 222         enum PVRSRV_ERROR eError;
 223
 224         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 225                               "SGX initialisation script part 1\n");
 226         eError =
 227             SGXRunScript(psDevInfo, psDevInfo->sScripts.asInitCommandsPart1,
 228                          SGX_MAX_INIT_COMMANDS);
 229         if (eError != PVRSRV_OK) {
 230                 PVR_DPF(PVR_DBG_ERROR,
 231                          "SGXInitialise: SGXRunScript (part 1) failed (%d)",
 232                          eError);
 233                 return PVRSRV_ERROR_GENERIC;
 234         }
 235         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 236                               "End of SGX initialisation script part 1\n");
 237
 238         SGXReset(psDevInfo, PDUMP_FLAGS_CONTINUOUS);
 239
 240
 241
 242         *psDevInfo->pui32KernelCCBEventKicker = 0;
 243 #if defined(PDUMP)
 244         PDUMPMEM(NULL, psDevInfo->psKernelCCBEventKickerMemInfo, 0,
 245                  sizeof(*psDevInfo->pui32KernelCCBEventKicker),
 246                  PDUMP_FLAGS_CONTINUOUS,
 247                  MAKEUNIQUETAG(psDevInfo->psKernelCCBEventKickerMemInfo));
 248 #endif
 249
 250         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 251                               "SGX initialisation script part 2\n");
 252         eError =
 253             SGXRunScript(psDevInfo, psDevInfo->sScripts.asInitCommandsPart2,
 254                          SGX_MAX_INIT_COMMANDS);
 255         if (eError != PVRSRV_OK) {
 256                 PVR_DPF(PVR_DBG_ERROR,
 257                          "SGXInitialise: SGXRunScript (part 2) failed (%d)",
 258                          eError);
 259                 return PVRSRV_ERROR_GENERIC;
 260         }
 261         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 262                               "End of SGX initialisation script part 2\n");
 263
 264         SGXStartTimer(psDevInfo, (IMG_BOOL)!bHardwareRecovery);
 265
 266         if (bHardwareRecovery) {
 267                 struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 268                     psDevInfo->psSGXHostCtl;
 269
 270                 if (PollForValueKM(&psSGXHostCtl->ui32InterruptClearFlags, 0,
 271                      PVRSRV_USSE_EDM_INTERRUPT_HWR,
 272                      MAX_HW_TIME_US / WAIT_TRY_COUNT, 1000) != PVRSRV_OK) {
 273                         PVR_DPF(PVR_DBG_ERROR, "SGXInitialise: "
 274                                         "Wait for uKernel HW Recovery failed");
 275                         PVR_DBG_BREAK;
 276                         return PVRSRV_ERROR_RETRY;
 277                 }
 278         }
 279
 280         PVR_ASSERT(psDevInfo->psKernelCCBCtl->ui32ReadOffset ==
 281                    psDevInfo->psKernelCCBCtl->ui32WriteOffset);
 282
 283         return PVRSRV_OK;
 284 }
 285
 286 enum PVRSRV_ERROR SGXDeinitialise(void *hDevCookie)
 287 {
 288         struct PVRSRV_SGXDEV_INFO *psDevInfo = (struct PVRSRV_SGXDEV_INFO *)
 289                                                                    hDevCookie;
 290         enum PVRSRV_ERROR eError;
 291
 292         if (psDevInfo->pvRegsBaseKM == NULL)
 293                 return PVRSRV_OK;
 294
 295         eError = SGXRunScript(psDevInfo, psDevInfo->sScripts.asDeinitCommands,
 296                          SGX_MAX_DEINIT_COMMANDS);
 297         if (eError != PVRSRV_OK) {
 298                 PVR_DPF(PVR_DBG_ERROR,
 299                          "SGXDeinitialise: SGXRunScript failed (%d)", eError);
 300                 return PVRSRV_ERROR_GENERIC;
 301         }
 302
 303         return PVRSRV_OK;
 304 }
 305
 306 static enum PVRSRV_ERROR DevInitSGXPart1(void *pvDeviceNode)
 307 {
 308         struct PVRSRV_SGXDEV_INFO *psDevInfo;
 309         void *hKernelDevMemContext;
 310         struct IMG_DEV_PHYADDR sPDDevPAddr;
 311         u32 i;
 312         struct PVRSRV_DEVICE_NODE *psDeviceNode = (struct PVRSRV_DEVICE_NODE *)
 313                                                                    pvDeviceNode;
 314         struct DEVICE_MEMORY_HEAP_INFO *psDeviceMemoryHeap =
 315             psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeap;
 316         enum PVRSRV_ERROR eError;
 317
 318         PDUMPCOMMENT("SGX Initialisation Part 1");
 319
 320         PDUMPCOMMENT("SGX Core Version Information: %s",
 321                      SGX_CORE_FRIENDLY_NAME);
 322         PDUMPCOMMENT("SGX Core Revision Information: multi rev support");
 323
 324         if (OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 325                        sizeof(struct PVRSRV_SGXDEV_INFO),
 326                        (void **)&psDevInfo, NULL) != PVRSRV_OK) {
 327                 PVR_DPF(PVR_DBG_ERROR,
 328                         "DevInitSGXPart1 : Failed to alloc memory for DevInfo");
 329                 return PVRSRV_ERROR_OUT_OF_MEMORY;
 330         }
 331         OSMemSet(psDevInfo, 0, sizeof(struct PVRSRV_SGXDEV_INFO));
 332
 333         psDevInfo->eDeviceType = DEV_DEVICE_TYPE;
 334         psDevInfo->eDeviceClass = DEV_DEVICE_CLASS;
 335
 336         psDeviceNode->pvDevice = (void *) psDevInfo;
 337
 338         psDevInfo->pvDeviceMemoryHeap = (void *) psDeviceMemoryHeap;
 339
 340         hKernelDevMemContext = BM_CreateContext(psDeviceNode, &sPDDevPAddr,
 341                                                 NULL, NULL);
 342
 343         psDevInfo->sKernelPDDevPAddr = sPDDevPAddr;
 344
 345         for (i = 0; i < psDeviceNode->sDevMemoryInfo.ui32HeapCount; i++) {
 346                 void *hDevMemHeap;
 347
 348                 switch (psDeviceMemoryHeap[i].DevMemHeapType) {
 349                 case DEVICE_MEMORY_HEAP_KERNEL:
 350                 case DEVICE_MEMORY_HEAP_SHARED:
 351                 case DEVICE_MEMORY_HEAP_SHARED_EXPORTED:
 352                         {
 353                                 hDevMemHeap =
 354                                     BM_CreateHeap(hKernelDevMemContext,
 355                                                   &psDeviceMemoryHeap[i]);
 356
 357                                 psDeviceMemoryHeap[i].hDevMemHeap = hDevMemHeap;
 358                                 break;
 359                         }
 360                 }
 361         }
 362
 363         eError = MMU_BIFResetPDAlloc(psDevInfo);
 364         if (eError != PVRSRV_OK) {
 365                 PVR_DPF(PVR_DBG_ERROR,
 366                          "DevInitSGX : Failed to alloc memory for BIF reset");
 367                 return PVRSRV_ERROR_GENERIC;
 368         }
 369
 370         return PVRSRV_OK;
 371 }
 372
 373 enum PVRSRV_ERROR SGXGetInfoForSrvinitKM(void *hDevHandle,
 374                                 struct SGX_BRIDGE_INFO_FOR_SRVINIT *psInitInfo)
 375 {
 376         struct PVRSRV_DEVICE_NODE *psDeviceNode;
 377         struct PVRSRV_SGXDEV_INFO *psDevInfo;
 378         enum PVRSRV_ERROR eError;
 379
 380         PDUMPCOMMENT("SGXGetInfoForSrvinit");
 381
 382         psDeviceNode = (struct PVRSRV_DEVICE_NODE *)hDevHandle;
 383         psDevInfo = (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 384
 385         psInitInfo->sPDDevPAddr = psDevInfo->sKernelPDDevPAddr;
 386
 387         eError =
 388             PVRSRVGetDeviceMemHeapsKM(hDevHandle, &psInitInfo->asHeapInfo[0]);
 389         if (eError != PVRSRV_OK) {
 390                 PVR_DPF(PVR_DBG_ERROR, "SGXGetInfoForSrvinit: "
 391                                 "PVRSRVGetDeviceMemHeapsKM failed (%d)",
 392                          eError);
 393                 return PVRSRV_ERROR_GENERIC;
 394         }
 395
 396         return eError;
 397 }
 398
 399 enum PVRSRV_ERROR DevInitSGXPart2KM(struct PVRSRV_PER_PROCESS_DATA *psPerProc,
 400                                    void *hDevHandle,
 401                                    struct SGX_BRIDGE_INIT_INFO *psInitInfo)
 402 {
 403         struct PVRSRV_DEVICE_NODE *psDeviceNode;
 404         struct PVRSRV_SGXDEV_INFO *psDevInfo;
 405         enum PVRSRV_ERROR eError;
 406         struct SGX_DEVICE_MAP *psSGXDeviceMap;
 407         enum PVR_POWER_STATE eDefaultPowerState;
 408         u32 l;
 409
 410         PDUMPCOMMENT("SGX Initialisation Part 2");
 411
 412         psDeviceNode = (struct PVRSRV_DEVICE_NODE *)hDevHandle;
 413         psDevInfo = (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 414
 415         eError = InitDevInfo(psPerProc, psDeviceNode, psInitInfo);
 416         if (eError != PVRSRV_OK) {
 417                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM: "
 418                                         "Failed to load EDM program");
 419                 goto failed_init_dev_info;
 420         }
 421
 422
 423         eError = SysGetDeviceMemoryMap(PVRSRV_DEVICE_TYPE_SGX,
 424                                        (void **) &psSGXDeviceMap);
 425         if (eError != PVRSRV_OK) {
 426                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM: "
 427                                         "Failed to get device memory map!");
 428                 return PVRSRV_ERROR_INIT_FAILURE;
 429         }
 430
 431         if (psSGXDeviceMap->pvRegsCpuVBase) {
 432                 psDevInfo->pvRegsBaseKM = psSGXDeviceMap->pvRegsCpuVBase;
 433         } else {
 434                 psDevInfo->pvRegsBaseKM =
 435                     OSMapPhysToLin(psSGXDeviceMap->sRegsCpuPBase,
 436                                    psSGXDeviceMap->ui32RegsSize,
 437                                    PVRSRV_HAP_KERNEL_ONLY | PVRSRV_HAP_UNCACHED,
 438                                    NULL);
 439                 if (!psDevInfo->pvRegsBaseKM) {
 440                         PVR_DPF(PVR_DBG_ERROR,
 441                                  "DevInitSGXPart2KM: Failed to map in regs\n");
 442                         return PVRSRV_ERROR_BAD_MAPPING;
 443                 }
 444         }
 445         psDevInfo->ui32RegSize = psSGXDeviceMap->ui32RegsSize;
 446         psDevInfo->sRegsPhysBase = psSGXDeviceMap->sRegsSysPBase;
 447
 448         psDeviceNode->pvISRData = psDeviceNode;
 449
 450         PVR_ASSERT(psDeviceNode->pfnDeviceISR == SGX_ISRHandler);
 451
 452         pvr_dev_lock();
 453         l = readl(&psDevInfo->psSGXHostCtl->ui32PowerStatus);
 454         l |= PVRSRV_USSE_EDM_POWMAN_NO_WORK;
 455         writel(l, &psDevInfo->psSGXHostCtl->ui32PowerStatus);
 456         pvr_dev_unlock();
 457         eDefaultPowerState = PVRSRV_POWER_STATE_D3;
 458
 459         eError = PVRSRVRegisterPowerDevice(psDeviceNode->sDevId.ui32DeviceIndex,
 460                                            SGXPrePowerStateExt,
 461                                            SGXPostPowerStateExt,
 462                                            SGXPreClockSpeedChange,
 463                                            SGXPostClockSpeedChange,
 464                                            (void *) psDeviceNode,
 465                                            PVRSRV_POWER_STATE_D3,
 466                                            eDefaultPowerState);
 467         if (eError != PVRSRV_OK) {
 468                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM: "
 469                                 "failed to register device with power manager");
 470                 return eError;
 471         }
 472
 473         OSMemSet(psDevInfo->psKernelCCB, 0,
 474                  sizeof(struct PVRSRV_SGX_KERNEL_CCB));
 475         OSMemSet(psDevInfo->psKernelCCBCtl, 0,
 476                  sizeof(struct PVRSRV_SGX_CCB_CTL));
 477         OSMemSet(psDevInfo->pui32KernelCCBEventKicker, 0,
 478                  sizeof(*psDevInfo->pui32KernelCCBEventKicker));
 479         PDUMPCOMMENT("Initialise Kernel CCB");
 480         PDUMPMEM(NULL, psDevInfo->psKernelCCBMemInfo, 0,
 481                  sizeof(struct PVRSRV_SGX_KERNEL_CCB), PDUMP_FLAGS_CONTINUOUS,
 482                  MAKEUNIQUETAG(psDevInfo->psKernelCCBMemInfo));
 483         PDUMPCOMMENT("Initialise Kernel CCB Control");
 484         PDUMPMEM(NULL, psDevInfo->psKernelCCBCtlMemInfo, 0,
 485                  sizeof(struct PVRSRV_SGX_CCB_CTL), PDUMP_FLAGS_CONTINUOUS,
 486                  MAKEUNIQUETAG(psDevInfo->psKernelCCBCtlMemInfo));
 487         PDUMPCOMMENT("Initialise Kernel CCB Event Kicker");
 488         PDUMPMEM(NULL, psDevInfo->psKernelCCBEventKickerMemInfo, 0,
 489                  sizeof(*psDevInfo->pui32KernelCCBEventKicker),
 490                  PDUMP_FLAGS_CONTINUOUS,
 491                  MAKEUNIQUETAG(psDevInfo->psKernelCCBEventKickerMemInfo));
 492
 493         psDevInfo->hTimer = SGXOSTimerInit(psDeviceNode);
 494         if (!psDevInfo->hTimer)
 495                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM : "
 496                         "Failed to initialize HW recovery timer");
 497
 498         return PVRSRV_OK;
 499
 500 failed_init_dev_info:
 501         return eError;
 502 }
 503
 504 static enum PVRSRV_ERROR DevDeInitSGX(void *pvDeviceNode)
 505 {
 506         struct PVRSRV_DEVICE_NODE *psDeviceNode =
 507                 (struct PVRSRV_DEVICE_NODE *)pvDeviceNode;
 508         struct PVRSRV_SGXDEV_INFO *psDevInfo =
 509                 (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 510         enum PVRSRV_ERROR eError;
 511         u32 ui32Heap;
 512         struct DEVICE_MEMORY_HEAP_INFO *psDeviceMemoryHeap;
 513         struct SGX_DEVICE_MAP *psSGXDeviceMap;
 514
 515         if (!psDevInfo) {
 516                 PVR_DPF(PVR_DBG_ERROR, "DevDeInitSGX: Null DevInfo");
 517                 return PVRSRV_OK;
 518         }
 519         if (psDevInfo->hTimer) {
 520                 SGXOSTimerCancel(psDevInfo->hTimer);
 521                 SGXOSTimerDeInit(psDevInfo->hTimer);
 522                 psDevInfo->hTimer = NULL;
 523         }
 524
 525         MMU_BIFResetPDFree(psDevInfo);
 526
 527         DeinitDevInfo(psDevInfo);
 528
 529
 530         psDeviceMemoryHeap =
 531             (struct DEVICE_MEMORY_HEAP_INFO *)psDevInfo->pvDeviceMemoryHeap;
 532         for (ui32Heap = 0;
 533              ui32Heap < psDeviceNode->sDevMemoryInfo.ui32HeapCount;
 534              ui32Heap++) {
 535                 switch (psDeviceMemoryHeap[ui32Heap].DevMemHeapType) {
 536                 case DEVICE_MEMORY_HEAP_KERNEL:
 537                 case DEVICE_MEMORY_HEAP_SHARED:
 538                 case DEVICE_MEMORY_HEAP_SHARED_EXPORTED:
 539                         {
 540                                 if (psDeviceMemoryHeap[ui32Heap].hDevMemHeap !=
 541                                     NULL)
 542                                         BM_DestroyHeap(psDeviceMemoryHeap
 543                                                        [ui32Heap].hDevMemHeap);
 544                                 break;
 545                         }
 546                 }
 547         }
 548
 549         if (!pvr_put_ctx(psDeviceNode->sDevMemoryInfo.pBMKernelContext))
 550                 pr_err("%s: kernel context still in use, can't free it",
 551                         __func__);
 552
 553         eError = PVRSRVRemovePowerDevice(
 554                                 ((struct PVRSRV_DEVICE_NODE *)pvDeviceNode)->
 555                                   sDevId.ui32DeviceIndex);
 556         if (eError != PVRSRV_OK)
 557                 return eError;
 558
 559         eError = SysGetDeviceMemoryMap(PVRSRV_DEVICE_TYPE_SGX,
 560                                        (void **)&psSGXDeviceMap);
 561         if (eError != PVRSRV_OK) {
 562                 PVR_DPF(PVR_DBG_ERROR,
 563                          "DevDeInitSGX: Failed to get device memory map!");
 564                 return eError;
 565         }
 566
 567         if (!psSGXDeviceMap->pvRegsCpuVBase)
 568                 if (psDevInfo->pvRegsBaseKM != NULL)
 569                         OSUnMapPhysToLin(psDevInfo->pvRegsBaseKM,
 570                                          psDevInfo->ui32RegSize,
 571                                          PVRSRV_HAP_KERNEL_ONLY |
 572                                                  PVRSRV_HAP_UNCACHED,
 573                                          NULL);
 574
 575         OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 576                   sizeof(struct PVRSRV_SGXDEV_INFO), psDevInfo, NULL);
 577
 578         psDeviceNode->pvDevice = NULL;
 579
 580         if (psDeviceMemoryHeap != NULL)
 581                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 582                           sizeof(struct DEVICE_MEMORY_HEAP_INFO) *
 583                           psDeviceNode->sDevMemoryInfo.ui32HeapCount,
 584                           psDeviceMemoryHeap, NULL);
 585
 586         return PVRSRV_OK;
 587 }
 588
 589 #if defined(PVRSRV_USSE_EDM_STATUS_DEBUG) || defined(CONFIG_DEBUG_FS)
 590
 591 static size_t __print_edm_trace(struct PVRSRV_SGXDEV_INFO *sdev, char *dst,
 592                                 size_t dst_len)
 593 {
 594         u32 *buf_start;
 595         u32 *buf_end;
 596         u32 *buf;
 597         size_t p = 0;
 598         size_t wr_ofs;
 599         int i;
 600
 601 #define _PR(fmt, ...) do {                                                 \
 602         if (dst)                                                           \
 603                 p += snprintf(dst + p, dst_len - p, fmt, ## __VA_ARGS__);  \
 604         else                                                               \
 605                 pr_err(fmt, ## __VA_ARGS__);                               \
 606 } while (0)
 607
 608         if (!sdev->psKernelEDMStatusBufferMemInfo)
 609                 return 0;
 610
 611         buf = sdev->psKernelEDMStatusBufferMemInfo->pvLinAddrKM;
 612
 613         _PR("Last SGX microkernel status code: 0x%x\n", *buf);
 614         buf++;
 615         wr_ofs = *buf;
 616         buf++;
 617
 618         buf_start = buf;
 619         buf_end = buf + SGXMK_TRACE_BUFFER_SIZE * 4;
 620
 621         buf += wr_ofs * 4;
 622
 623         /* Dump the status values */
 624         for (i = 0; i < SGXMK_TRACE_BUFFER_SIZE; i++) {
 625                 _PR("%3d %08X %08X %08X %08X\n",
 626                     i, buf[2], buf[3], buf[1], buf[0]);
 627                 buf += 4;
 628                 if (buf >= buf_end)
 629                         buf = buf_start;
 630         }
 631
 632         return p > dst_len ? dst_len : p;
 633 #undef _PR
 634 }
 635
 636 size_t snprint_edm_trace(struct PVRSRV_SGXDEV_INFO *sdev, char *buf,
 637                          size_t buf_size)
 638 {
 639         return __print_edm_trace(sdev, buf, buf_size);
 640 }
 641
 642 #endif
 643
 644 static void dump_edm(struct PVRSRV_SGXDEV_INFO *sdev)
 645 {
 646 #ifdef PVRSRV_USSE_EDM_STATUS_DEBUG
 647         __print_edm_trace(sdev, NULL, 0);
 648 #endif
 649 }
 650
 651 static void dump_process_info(struct PVRSRV_DEVICE_NODE *dev)
 652 {
 653         struct PVRSRV_SGXDEV_INFO *dev_info = dev->pvDevice;
 654         u32 page_dir = readl(dev_info->pvRegsBaseKM +
 655                                 EUR_CR_BIF_DIR_LIST_BASE0);
 656         struct BM_CONTEXT *bm_ctx;
 657         struct RESMAN_CONTEXT *res_ctx = NULL;
 658
 659         bm_ctx = bm_find_context(dev->sDevMemoryInfo.pBMContext, page_dir);
 660         if (bm_ctx)
 661                 res_ctx = pvr_get_resman_ctx(bm_ctx);
 662
 663         if (res_ctx) {
 664                 struct task_struct *tsk;
 665                 struct PVRSRV_PER_PROCESS_DATA *proc;
 666                 int pid;
 667
 668                 proc = pvr_get_proc_by_ctx(res_ctx);
 669                 pid = proc->ui32PID;
 670                 rcu_read_lock();
 671                 tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
 672                 pr_err("PID = %d, process name = %s\n", pid, tsk->comm);
 673                 rcu_read_unlock();
 674         }
 675 }
 676
 677 static void dump_sgx_registers(struct PVRSRV_SGXDEV_INFO *psDevInfo)
 678 {
 679         pr_err("EVENT_STATUS =     0x%08X\n"
 680                 "EVENT_STATUS2 =    0x%08X\n"
 681                 "BIF_CTRL =         0x%08X\n"
 682                 "BIF_INT_STAT =     0x%08X\n"
 683                 "BIF_MEM_REQ_STAT = 0x%08X\n"
 684                 "BIF_FAULT  =       0x%08X\n"
 685                 "CLKGATECTL =       0x%08X\n",
 686                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_EVENT_STATUS),
 687                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_EVENT_STATUS2),
 688                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_CTRL),
 689                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_INT_STAT),
 690                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_MEM_REQ_STAT),
 691                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_FAULT),
 692                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_CLKGATECTL));
 693 }
 694
 695 /* Should be called with pvr_lock held */
 696 void HWRecoveryResetSGX(struct PVRSRV_DEVICE_NODE *psDeviceNode)
 697 {
 698         enum PVRSRV_ERROR eError;
 699         struct PVRSRV_SGXDEV_INFO *psDevInfo =
 700             (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 701         struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 702                                         psDevInfo->psSGXHostCtl;
 703         u32 l;
 704         int max_retries = 10;
 705
 706         BUG_ON(!pvr_is_locked());
 707
 708         l = readl(&psSGXHostCtl->ui32InterruptClearFlags);
 709         l |= PVRSRV_USSE_EDM_INTERRUPT_HWR;
 710         writel(l, &psSGXHostCtl->ui32InterruptClearFlags);
 711
 712         pr_err("%s: SGX Hardware Recovery triggered\n", __func__);
 713
 714         dump_process_info(psDeviceNode);
 715         dump_sgx_registers(psDevInfo);
 716         dump_edm(psDevInfo);
 717
 718         PDUMPSUSPEND();
 719
 720         do {
 721                 eError = SGXInitialise(psDevInfo, IMG_TRUE);
 722                 if (eError != PVRSRV_ERROR_RETRY)
 723                         break;
 724         } while (max_retries--);
 725
 726         if (eError != PVRSRV_OK) {
 727                 pr_err("%s: recovery failed (%d). Disabling the driver",
 728                         __func__, eError);
 729                 pvr_disable();
 730
 731                 PDUMPRESUME();
 732
 733                 return;
 734         }
 735
 736         PDUMPRESUME();
 737
 738         SGXScheduleProcessQueues(psDeviceNode);
 739
 740         PVRSRVProcessQueues(IMG_TRUE);
 741 }
 742
 743 static unsigned long sgx_reset_forced;
 744
 745 static void SGXOSTimer(struct work_struct *work)
 746 {
 747         struct timer_work_data *data = container_of(work,
 748                                                     struct timer_work_data,
 749                                                     work.work);
 750         struct PVRSRV_DEVICE_NODE *psDeviceNode = data->psDeviceNode;
 751         struct PVRSRV_SGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
 752         static u32 ui32EDMTasks;
 753         static u32 ui32LockupCounter;
 754         static u32 ui32NumResets;
 755         u32 ui32CurrentEDMTasks;
 756         IMG_BOOL bLockup = IMG_FALSE;
 757         IMG_BOOL bPoweredDown;
 758
 759         pvr_lock();
 760
 761         if (!data->armed || pvr_is_disabled()) {
 762                 pvr_unlock();
 763                 return;
 764         }
 765
 766         psDevInfo->ui32TimeStamp++;
 767
 768 #if defined(NO_HARDWARE)
 769         bPoweredDown = IMG_TRUE;
 770 #else
 771         bPoweredDown = (IMG_BOOL) !SGXIsDevicePowered(psDeviceNode);
 772 #endif
 773
 774         if (bPoweredDown) {
 775                 ui32LockupCounter = 0;
 776         } else {
 777                 pvr_dev_lock();
 778                 ui32CurrentEDMTasks = OSReadHWReg(psDevInfo->pvRegsBaseKM,
 779                                                 psDevInfo->ui32EDMTaskReg0);
 780                 if (psDevInfo->ui32EDMTaskReg1 != 0)
 781                         ui32CurrentEDMTasks ^=
 782                             OSReadHWReg(psDevInfo->pvRegsBaseKM,
 783                                         psDevInfo->ui32EDMTaskReg1);
 784                 if ((ui32CurrentEDMTasks == ui32EDMTasks) &&
 785                     (psDevInfo->ui32NumResets == ui32NumResets)) {
 786                         ui32LockupCounter++;
 787                         if (ui32LockupCounter == 3) {
 788                                 ui32LockupCounter = 0;
 789                                 PVR_DPF(PVR_DBG_ERROR, "SGXOSTimer() "
 790                                         "detected SGX lockup (0x%x tasks)",
 791                                          ui32EDMTasks);
 792
 793                                 bLockup = IMG_TRUE;
 794                         }
 795                 } else {
 796                         ui32LockupCounter = 0;
 797                         ui32EDMTasks = ui32CurrentEDMTasks;
 798                         ui32NumResets = psDevInfo->ui32NumResets;
 799                 }
 800                 pvr_dev_unlock();
 801         }
 802
 803         bLockup |= cmpxchg(&sgx_reset_forced, 1, 0);
 804
 805         if (bLockup) {
 806                 struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 807                                                 psDevInfo->psSGXHostCtl;
 808                 u32 l;
 809
 810                 pvr_dev_lock();
 811                 l = readl(&psSGXHostCtl->ui32HostDetectedLockups);
 812                 l++;
 813                 writel(l, &psSGXHostCtl->ui32HostDetectedLockups);
 814
 815                 HWRecoveryResetSGX(psDeviceNode);
 816                 pvr_dev_unlock();
 817         }
 818
 819         queue_delayed_work(data->work_queue, &data->work,
 820                            msecs_to_jiffies(data->interval));
 821
 822         pvr_unlock();
 823 }
 824
 825 struct timer_work_data *
 826 SGXOSTimerInit(struct PVRSRV_DEVICE_NODE *psDeviceNode)
 827 {
 828         struct timer_work_data *data;
 829
 830         data = kzalloc(sizeof(struct timer_work_data), GFP_KERNEL);
 831         if (!data)
 832                 return NULL;
 833
 834         data->work_queue = create_workqueue("SGXOSTimer");
 835         if (!data->work_queue) {
 836                 kfree(data);
 837                 return NULL;
 838         }
 839
 840         data->interval = 150;
 841         data->psDeviceNode = psDeviceNode;
 842         INIT_DELAYED_WORK(&data->work, SGXOSTimer);
 843
 844         return data;
 845 }
 846
 847 void SGXOSTimerDeInit(struct timer_work_data *data)
 848 {
 849         data->armed = false;
 850         destroy_workqueue(data->work_queue);
 851         kfree(data);
 852 }
 853
 854 enum PVRSRV_ERROR SGXOSTimerEnable(struct timer_work_data *data)
 855 {
 856         if (!data)
 857                 return PVRSRV_ERROR_GENERIC;
 858
 859         if (queue_delayed_work(data->work_queue, &data->work,
 860                                msecs_to_jiffies(data->interval))) {
 861                 data->armed = true;
 862                 return PVRSRV_OK;
 863         }
 864
 865         return PVRSRV_ERROR_GENERIC;
 866 }
 867
 868 enum PVRSRV_ERROR SGXOSTimerCancel(struct timer_work_data *data)
 869 {
 870         if (!data)
 871                 return PVRSRV_ERROR_GENERIC;
 872
 873         data->armed = false;
 874         cancel_delayed_work(&data->work);
 875
 876         return PVRSRV_OK;
 877 }
 878
 879 int sgx_force_reset(void)
 880 {
 881         return !cmpxchg(&sgx_reset_forced, 0, 1);
 882 }
 883
 884 static IMG_BOOL SGX_ISRHandler(void *pvData)
 885 {
 886         IMG_BOOL bInterruptProcessed = IMG_FALSE;
 887
 888         {
 889                 u32 ui32EventStatus, ui32EventEnable;
 890                 u32 ui32EventClear = 0;
 891                 struct PVRSRV_DEVICE_NODE *psDeviceNode;
 892                 struct PVRSRV_SGXDEV_INFO *psDevInfo;
 893
 894                 if (pvData == NULL) {
 895                         PVR_DPF(PVR_DBG_ERROR,
 896                                  "SGX_ISRHandler: Invalid params\n");
 897                         return bInterruptProcessed;
 898                 }
 899
 900                 psDeviceNode = (struct PVRSRV_DEVICE_NODE *)pvData;
 901                 psDevInfo = (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 902
 903                 ui32EventStatus =
 904                     OSReadHWReg(psDevInfo->pvRegsBaseKM, EUR_CR_EVENT_STATUS);
 905                 ui32EventEnable = OSReadHWReg(psDevInfo->pvRegsBaseKM,
 906                                                 EUR_CR_EVENT_HOST_ENABLE);
 907
 908                 gui32EventStatusServicesByISR = ui32EventStatus;
 909
 910                 ui32EventStatus &= ui32EventEnable;
 911
 912                 if (ui32EventStatus & EUR_CR_EVENT_STATUS_SW_EVENT_MASK)
 913                         ui32EventClear |= EUR_CR_EVENT_HOST_CLEAR_SW_EVENT_MASK;
 914
 915                 if (ui32EventClear) {
 916                         bInterruptProcessed = IMG_TRUE;
 917
 918                         ui32EventClear |=
 919                             EUR_CR_EVENT_HOST_CLEAR_MASTER_INTERRUPT_MASK;
 920
 921                         OSWriteHWReg(psDevInfo->pvRegsBaseKM,
 922                                      EUR_CR_EVENT_HOST_CLEAR, ui32EventClear);
 923                 }
 924         }
 925
 926         return bInterruptProcessed;
 927 }
 928
 929 static void SGX_MISRHandler(void *pvData)
 930 {
 931         struct PVRSRV_DEVICE_NODE *psDeviceNode =
 932                         (struct PVRSRV_DEVICE_NODE *)pvData;
 933         struct PVRSRV_SGXDEV_INFO *psDevInfo =
 934                         (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 935         struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 936                         psDevInfo->psSGXHostCtl;
 937         u32 l1, l2;
 938         int dev_idx;
 939         enum PVRSRV_ERROR err;
 940
 941         dev_idx = psDeviceNode->sDevId.ui32DeviceIndex;
 942
 943         pvr_dev_lock();
 944
 945         err = PVRSRVSetDevicePowerStateKM(dev_idx, PVRSRV_POWER_STATE_D0);
 946         BUG_ON(err != PVRSRV_OK);
 947
 948         l1 = readl(&psSGXHostCtl->ui32InterruptFlags);
 949         l2 = readl(&psSGXHostCtl->ui32InterruptClearFlags);
 950
 951         if ((l1 & PVRSRV_USSE_EDM_INTERRUPT_HWR) &&
 952             !(l2 & PVRSRV_USSE_EDM_INTERRUPT_HWR))
 953                 HWRecoveryResetSGX(psDeviceNode);
 954
 955         if (psDeviceNode->bReProcessDeviceCommandComplete)
 956                 SGXScheduleProcessQueues(psDeviceNode);
 957
 958         SGXTestActivePowerEvent(psDeviceNode);
 959
 960         pvr_dev_unlock();
 961 }
 962
 963 enum PVRSRV_ERROR SGXRegisterDevice(struct PVRSRV_DEVICE_NODE *psDeviceNode)
 964 {
 965         struct DEVICE_MEMORY_INFO *psDevMemoryInfo;
 966         struct DEVICE_MEMORY_HEAP_INFO *psDeviceMemoryHeap;
 967
 968         psDeviceNode->sDevId.eDeviceType = DEV_DEVICE_TYPE;
 969         psDeviceNode->sDevId.eDeviceClass = DEV_DEVICE_CLASS;
 970
 971         psDeviceNode->pfnInitDevice = DevInitSGXPart1;
 972         psDeviceNode->pfnDeInitDevice = DevDeInitSGX;
 973
 974         psDeviceNode->pfnInitDeviceCompatCheck = SGXDevInitCompatCheck;
 975
 976         psDeviceNode->pfnMMUInitialise = MMU_Initialise;
 977         psDeviceNode->pfnMMUFinalise = MMU_Finalise;
 978         psDeviceNode->pfnMMUInsertHeap = MMU_InsertHeap;
 979         psDeviceNode->pfnMMUCreate = MMU_Create;
 980         psDeviceNode->pfnMMUDelete = MMU_Delete;
 981         psDeviceNode->pfnMMUAlloc = MMU_Alloc;
 982         psDeviceNode->pfnMMUFree = MMU_Free;
 983         psDeviceNode->pfnMMUMapPages = MMU_MapPages;
 984         psDeviceNode->pfnMMUMapShadow = MMU_MapShadow;
 985         psDeviceNode->pfnMMUUnmapPages = MMU_UnmapPages;
 986         psDeviceNode->pfnMMUMapScatter = MMU_MapScatter;
 987         psDeviceNode->pfnMMUGetPhysPageAddr = MMU_GetPhysPageAddr;
 988         psDeviceNode->pfnMMUGetPDDevPAddr = MMU_GetPDDevPAddr;
 989
 990         psDeviceNode->pfnDeviceISR = SGX_ISRHandler;
 991         psDeviceNode->pfnDeviceMISR = SGX_MISRHandler;
 992
 993         psDeviceNode->pfnDeviceCommandComplete = SGXCommandComplete;
 994
 995         psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
 996
 997         psDevMemoryInfo->ui32AddressSpaceSizeLog2 =
 998             SGX_FEATURE_ADDRESS_SPACE_SIZE;
 999
1000         psDevMemoryInfo->ui32Flags = 0;
1001         psDevMemoryInfo->ui32HeapCount = SGX_MAX_HEAP_ID;
1002         psDevMemoryInfo->ui32SyncHeapID = SGX_SYNCINFO_HEAP_ID;
1003
1004         psDevMemoryInfo->ui32MappingHeapID = SGX_GENERAL_HEAP_ID;
1005
1006         if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
1007                        sizeof(struct DEVICE_MEMORY_HEAP_INFO) *
1008                        psDevMemoryInfo->ui32HeapCount,
1009                        (void **) &psDevMemoryInfo->psDeviceMemoryHeap,
1010                        NULL) != PVRSRV_OK) {
1011                 PVR_DPF(PVR_DBG_ERROR, "SGXRegisterDevice : "
1012                                 "Failed to alloc memory for "
1013                                 "struct DEVICE_MEMORY_HEAP_INFO");
1014                 return PVRSRV_ERROR_OUT_OF_MEMORY;
1015         }
1016         OSMemSet(psDevMemoryInfo->psDeviceMemoryHeap, 0,
1017                  sizeof(struct DEVICE_MEMORY_HEAP_INFO) *
1018                  psDevMemoryInfo->ui32HeapCount);
1019
1020         psDeviceMemoryHeap = psDevMemoryInfo->psDeviceMemoryHeap;
1021
1022         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32HeapID =
1023             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_GENERAL_HEAP_ID);
1024         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].sDevVAddrBase.uiAddr =
1025             SGX_GENERAL_HEAP_BASE;
1026         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32HeapSize =
1027             SGX_GENERAL_HEAP_SIZE;
1028         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32Attribs =
1029             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1030             PVRSRV_HAP_SINGLE_PROCESS;
1031         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].pszName = "General";
1032         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].pszBSName = "General BS";
1033         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].DevMemHeapType =
1034             DEVICE_MEMORY_HEAP_PERCONTEXT;
1035
1036         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32DataPageSize =
1037             SGX_MMU_PAGE_SIZE;
1038
1039         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32HeapID =
1040             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_TADATA_HEAP_ID);
1041         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].sDevVAddrBase.uiAddr =
1042             SGX_TADATA_HEAP_BASE;
1043         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32HeapSize =
1044             SGX_TADATA_HEAP_SIZE;
1045         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32Attribs =
1046             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION
1047             | PVRSRV_HAP_MULTI_PROCESS;
1048         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].pszName = "TA Data";
1049         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].pszBSName = "TA Data BS";
1050         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].DevMemHeapType =
1051             DEVICE_MEMORY_HEAP_PERCONTEXT;
1052
1053         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32DataPageSize =
1054             SGX_MMU_PAGE_SIZE;
1055
1056         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32HeapID =
1057             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_KERNEL_CODE_HEAP_ID);
1058         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].sDevVAddrBase.uiAddr =
1059             SGX_KERNEL_CODE_HEAP_BASE;
1060         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32HeapSize =
1061             SGX_KERNEL_CODE_HEAP_SIZE;
1062         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32Attribs =
1063             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1064             PVRSRV_HAP_MULTI_PROCESS;
1065         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].pszName = "Kernel Code";
1066         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].pszBSName =
1067             "Kernel Code BS";
1068         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].DevMemHeapType =
1069             DEVICE_MEMORY_HEAP_SHARED_EXPORTED;
1070
1071         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32DataPageSize =
1072             SGX_MMU_PAGE_SIZE;
1073
1074         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32HeapID =
1075             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_KERNEL_DATA_HEAP_ID);
1076         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].sDevVAddrBase.uiAddr =
1077             SGX_KERNEL_DATA_HEAP_BASE;
1078         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32HeapSize =
1079             SGX_KERNEL_DATA_HEAP_SIZE;
1080         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32Attribs =
1081             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1082             PVRSRV_HAP_MULTI_PROCESS;
1083         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].pszName = "KernelData";
1084         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].pszBSName = "KernelData BS";
1085         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].DevMemHeapType =
1086             DEVICE_MEMORY_HEAP_SHARED_EXPORTED;
1087
1088         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32DataPageSize =
1089             SGX_MMU_PAGE_SIZE;
1090
1091         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32HeapID =
1092             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_PIXELSHADER_HEAP_ID);
1093         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].sDevVAddrBase.uiAddr =
1094             SGX_PIXELSHADER_HEAP_BASE;
1095         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32HeapSize =
1096             SGX_PIXELSHADER_HEAP_SIZE;
1097         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32Attribs =
1098             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1099             PVRSRV_HAP_SINGLE_PROCESS;
1100         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].pszName = "PixelShaderUSSE";
1101         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].pszBSName =
1102             "PixelShaderUSSE BS";
1103         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].DevMemHeapType =
1104             DEVICE_MEMORY_HEAP_PERCONTEXT;
1105
1106         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32DataPageSize =
1107             SGX_MMU_PAGE_SIZE;
1108
1109         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32HeapID =
1110             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_VERTEXSHADER_HEAP_ID);
1111         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].sDevVAddrBase.uiAddr =
1112             SGX_VERTEXSHADER_HEAP_BASE;
1113         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32HeapSize =
1114             SGX_VERTEXSHADER_HEAP_SIZE;
1115         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32Attribs =
1116             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1117             PVRSRV_HAP_SINGLE_PROCESS;
1118         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].pszName =
1119             "VertexShaderUSSE";
1120         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].pszBSName =
1121             "VertexShaderUSSE BS";
1122         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].DevMemHeapType =
1123             DEVICE_MEMORY_HEAP_PERCONTEXT;
1124
1125         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32DataPageSize =
1126             SGX_MMU_PAGE_SIZE;
1127
1128         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32HeapID =
1129             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_PDSPIXEL_CODEDATA_HEAP_ID);
1130         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].sDevVAddrBase.uiAddr =
1131             SGX_PDSPIXEL_CODEDATA_HEAP_BASE;
1132         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32HeapSize =
1133             SGX_PDSPIXEL_CODEDATA_HEAP_SIZE;
1134         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32Attribs =
1135             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1136             PVRSRV_HAP_SINGLE_PROCESS;
1137         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].pszName =
1138             "PDSPixelCodeData";
1139         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].pszBSName =
1140             "PDSPixelCodeData BS";
1141         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].DevMemHeapType =
1142             DEVICE_MEMORY_HEAP_PERCONTEXT;
1143
1144         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32DataPageSize =
1145             SGX_MMU_PAGE_SIZE;
1146
1147         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32HeapID =
1148             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_PDSVERTEX_CODEDATA_HEAP_ID);
1149         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].sDevVAddrBase.
1150             uiAddr = SGX_PDSVERTEX_CODEDATA_HEAP_BASE;
1151         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32HeapSize =
1152             SGX_PDSVERTEX_CODEDATA_HEAP_SIZE;
1153         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32Attribs =
1154             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1155             PVRSRV_HAP_SINGLE_PROCESS;
1156         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].pszName =
1157             "PDSVertexCodeData";
1158         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].pszBSName =
1159             "PDSVertexCodeData BS";
1160         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].DevMemHeapType =
1161             DEVICE_MEMORY_HEAP_PERCONTEXT;
1162
1163         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32DataPageSize =
1164             SGX_MMU_PAGE_SIZE;
1165
1166         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32HeapID =
1167             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_SYNCINFO_HEAP_ID);
1168         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].sDevVAddrBase.uiAddr =
1169             SGX_SYNCINFO_HEAP_BASE;
1170         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32HeapSize =
1171             SGX_SYNCINFO_HEAP_SIZE;
1172
1173         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32Attribs =
1174             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1175             PVRSRV_HAP_MULTI_PROCESS;
1176         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].pszName = "CacheCoherent";
1177         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].pszBSName = "CacheCoherent BS";
1178
1179         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].DevMemHeapType =
1180             DEVICE_MEMORY_HEAP_SHARED_EXPORTED;
1181
1182         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32DataPageSize =
1183             SGX_MMU_PAGE_SIZE;
1184
1185         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32HeapID =
1186             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_3DPARAMETERS_HEAP_ID);
1187         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].sDevVAddrBase.uiAddr =
1188             SGX_3DPARAMETERS_HEAP_BASE;
1189         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32HeapSize =
1190             SGX_3DPARAMETERS_HEAP_SIZE;
1191         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].pszName = "3DParameters";
1192         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].pszBSName =
1193             "3DParameters BS";
1194         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32Attribs =
1195             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1196             PVRSRV_HAP_SINGLE_PROCESS;
1197         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].DevMemHeapType =
1198             DEVICE_MEMORY_HEAP_PERCONTEXT;
1199
1200         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32DataPageSize =
1201             SGX_MMU_PAGE_SIZE;
1202
1203         return PVRSRV_OK;
1204 }
1205
1206 enum PVRSRV_ERROR SGXGetClientInfoKM(void *hDevCookie,
1207                                          struct SGX_CLIENT_INFO *psClientInfo)
1208 {
1209         struct PVRSRV_SGXDEV_INFO *psDevInfo =
1210             (struct PVRSRV_SGXDEV_INFO *)
1211                         ((struct PVRSRV_DEVICE_NODE *)hDevCookie)->pvDevice;
1212
1213         psDevInfo->ui32ClientRefCount++;
1214 #ifdef PDUMP
1215         if (psDevInfo->ui32ClientRefCount == 1)
1216                 psDevInfo->psKernelCCBInfo->ui32CCBDumpWOff = 0;
1217 #endif
1218         psClientInfo->ui32ProcessID = OSGetCurrentProcessIDKM();
1219
1220         OSMemCopy(&psClientInfo->asDevData, &psDevInfo->asSGXDevData,
1221                   sizeof(psClientInfo->asDevData));
1222
1223         return PVRSRV_OK;
1224 }
1225
1226 enum PVRSRV_ERROR SGXDevInitCompatCheck(struct PVRSRV_DEVICE_NODE *psDeviceNode)
1227 {
1228         struct PVRSRV_SGXDEV_INFO *psDevInfo;
1229         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo;
1230         enum PVRSRV_ERROR eError;
1231 #if !defined(NO_HARDWARE)
1232         u32 opts;
1233         u32 opt_mismatch;
1234         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1235 #endif
1236
1237         if (psDeviceNode->sDevId.eDeviceType != PVRSRV_DEVICE_TYPE_SGX) {
1238                 PVR_DPF(PVR_DBG_ERROR,
1239                          "SGXDevInitCompatCheck: Device not of type SGX");
1240                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1241                 goto exit;
1242         }
1243         psDevInfo = psDeviceNode->pvDevice;
1244         psMemInfo = psDevInfo->psKernelSGXMiscMemInfo;
1245
1246 #if !defined(NO_HARDWARE)
1247
1248         eError = SGXGetBuildInfoKM(psDevInfo, psDeviceNode);
1249         if (eError != PVRSRV_OK) {
1250                 pr_err("pvr: unable to validate device DDK version\n");
1251                 goto exit;
1252         }
1253         psSGXFeatures =
1254             &((struct PVRSRV_SGX_MISCINFO_INFO *)(psMemInfo->pvLinAddrKM))->
1255                                                             sSGXFeatures;
1256         if ((psSGXFeatures->ui32DDKVersion !=
1257              ((PVRVERSION_MAJ << 16) | (PVRVERSION_MIN << 8) |
1258               PVRVERSION_BRANCH)) ||
1259              (psSGXFeatures->ui32DDKBuild != PVRVERSION_BUILD)) {
1260                 pr_err("pvr: incompatible driver DDK revision (%d)"
1261                         "/device DDK revision (%d).\n",
1262                          PVRVERSION_BUILD, psSGXFeatures->ui32DDKBuild);
1263                 eError = PVRSRV_ERROR_DDK_VERSION_MISMATCH;
1264                 goto exit;
1265         } else {
1266                 PVR_DPF(PVR_DBG_WARNING, "(Success) SGXInit: "
1267                                 "driver DDK (%ld) and device DDK (%ld) match",
1268                          PVRVERSION_BUILD, psSGXFeatures->ui32DDKBuild);
1269         }
1270
1271         opts = psSGXFeatures->ui32BuildOptions;
1272         opt_mismatch = opts ^ SGX_BUILD_OPTIONS;
1273         /* we support the ABIs both with and without EDM tracing option */
1274         opt_mismatch &= ~PVRSRV_USSE_EDM_STATUS_DEBUG_SET_OFFSET;
1275         if (opt_mismatch) {
1276                 if (SGX_BUILD_OPTIONS & opt_mismatch)
1277                         pr_err("pvr: mismatch in driver and microkernel build "
1278                                 "options; extra options present in driver: "
1279                                 "(0x%x)", SGX_BUILD_OPTIONS & opt_mismatch);
1280
1281                 if (opts & opt_mismatch)
1282                         pr_err("pvr: Mismatch in driver and microkernel build "
1283                                 "options; extra options present in "
1284                                 "microkernel: (0x%x)", opts & opt_mismatch);
1285                 eError = PVRSRV_ERROR_BUILD_MISMATCH;
1286                 goto exit;
1287         } else {
1288                 PVR_DPF(PVR_DBG_WARNING, "(Success) SGXInit: "
1289                                 "Driver and microkernel build options match.");
1290         }
1291
1292 #endif
1293         eError = PVRSRV_OK;
1294 exit:
1295         return eError;
1296 }
1297
1298 static
1299 enum PVRSRV_ERROR SGXGetBuildInfoKM(struct PVRSRV_SGXDEV_INFO *psDevInfo,
1300                                     struct PVRSRV_DEVICE_NODE *psDeviceNode)
1301 {
1302         enum PVRSRV_ERROR eError;
1303         struct SGXMKIF_COMMAND sCommandData;
1304         struct PVRSRV_SGX_MISCINFO_INFO *psSGXMiscInfoInt;
1305         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1306
1307         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo =
1308             psDevInfo->psKernelSGXMiscMemInfo;
1309
1310         if (!psMemInfo->pvLinAddrKM) {
1311                 PVR_DPF(PVR_DBG_ERROR, "SGXGetMiscInfoKM: Invalid address.");
1312                 return PVRSRV_ERROR_INVALID_PARAMS;
1313         }
1314         psSGXMiscInfoInt = psMemInfo->pvLinAddrKM;
1315         psSGXMiscInfoInt->ui32MiscInfoFlags &= ~PVRSRV_USSE_MISCINFO_READY;
1316         psSGXFeatures = &psSGXMiscInfoInt->sSGXFeatures;
1317
1318         OSMemSet(psMemInfo->pvLinAddrKM, 0,
1319                  sizeof(struct PVRSRV_SGX_MISCINFO_INFO));
1320
1321         sCommandData.ui32Data[1] = psMemInfo->sDevVAddr.uiAddr;
1322
1323         OSMemSet(psSGXFeatures, 0, sizeof(*psSGXFeatures));
1324
1325         mb();
1326
1327         eError = SGXScheduleCCBCommandKM(psDeviceNode,
1328                                          SGXMKIF_COMMAND_REQUEST_SGXMISCINFO,
1329                                          &sCommandData, KERNEL_ID, 0);
1330
1331         if (eError != PVRSRV_OK) {
1332                 PVR_DPF(PVR_DBG_ERROR,
1333                          "SGXGetMiscInfoKM: SGXScheduleCCBCommandKM failed.");
1334                 return eError;
1335         }
1336
1337 #if !defined(NO_HARDWARE)
1338         {
1339                 IMG_BOOL bTimeout = IMG_TRUE;
1340
1341                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US) {
1342                         if ((psSGXMiscInfoInt->
1343                              ui32MiscInfoFlags & PVRSRV_USSE_MISCINFO_READY) !=
1344                             0) {
1345                                 bTimeout = IMG_FALSE;
1346                                 break;
1347                         }
1348                 }
1349                 END_LOOP_UNTIL_TIMEOUT();
1350
1351                 if (bTimeout)
1352                         return PVRSRV_ERROR_TIMEOUT;
1353         }
1354 #endif
1355
1356         return PVRSRV_OK;
1357 }
1358
1359 enum PVRSRV_ERROR SGXGetMiscInfoKM(struct PVRSRV_SGXDEV_INFO *psDevInfo,
1360                                        struct SGX_MISC_INFO *psMiscInfo,
1361                                        struct PVRSRV_DEVICE_NODE *psDeviceNode)
1362 {
1363         switch (psMiscInfo->eRequest) {
1364         case SGX_MISC_INFO_REQUEST_CLOCKSPEED:
1365                 {
1366                         psMiscInfo->uData.ui32SGXClockSpeed =
1367                             psDevInfo->ui32CoreClockSpeed;
1368                         return PVRSRV_OK;
1369                 }
1370
1371         case SGX_MISC_INFO_REQUEST_SGXREV:
1372                 {
1373                         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1374                         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo =
1375                             psDevInfo->psKernelSGXMiscMemInfo;
1376
1377                         SGXGetBuildInfoKM(psDevInfo, psDeviceNode);
1378                         psSGXFeatures =
1379                             &((struct PVRSRV_SGX_MISCINFO_INFO *)(psMemInfo->
1380                                                   pvLinAddrKM))->sSGXFeatures;
1381
1382                         psMiscInfo->uData.sSGXFeatures = *psSGXFeatures;
1383
1384                         PVR_DPF(PVR_DBG_MESSAGE, "SGXGetMiscInfoKM: "
1385                                         "Core 0x%lx, sw ID 0x%lx, "
1386                                         "sw Rev 0x%lx\n",
1387                                  psSGXFeatures->ui32CoreRev,
1388                                  psSGXFeatures->ui32CoreIdSW,
1389                                  psSGXFeatures->ui32CoreRevSW);
1390                         PVR_DPF(PVR_DBG_MESSAGE, "SGXGetMiscInfoKM: "
1391                                         "DDK version 0x%lx, DDK build 0x%lx\n",
1392                                  psSGXFeatures->ui32DDKVersion,
1393                                  psSGXFeatures->ui32DDKBuild);
1394
1395                         return PVRSRV_OK;
1396                 }
1397
1398         case SGX_MISC_INFO_REQUEST_DRIVER_SGXREV:
1399                 {
1400                         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo =
1401                             psDevInfo->psKernelSGXMiscMemInfo;
1402                         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1403
1404                         psSGXFeatures = &((struct PVRSRV_SGX_MISCINFO_INFO *)(
1405                                         psMemInfo->pvLinAddrKM))->sSGXFeatures;
1406
1407                         OSMemSet(psMemInfo->pvLinAddrKM, 0,
1408                                  sizeof(struct PVRSRV_SGX_MISCINFO_INFO));
1409
1410                         psSGXFeatures->ui32DDKVersion =
1411                             (PVRVERSION_MAJ << 16) |
1412                             (PVRVERSION_MIN << 8) | PVRVERSION_BRANCH;
1413                         psSGXFeatures->ui32DDKBuild = PVRVERSION_BUILD;
1414
1415                         psMiscInfo->uData.sSGXFeatures = *psSGXFeatures;
1416                         return PVRSRV_OK;
1417                 }
1418
1419         case SGX_MISC_INFO_REQUEST_SET_HWPERF_STATUS:
1420                 {
1421                         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1422                             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1423                         unsigned ui32MatchingFlags;
1424
1425                         if ((psMiscInfo->uData.ui32NewHWPerfStatus &
1426                              ~(PVRSRV_SGX_HWPERF_GRAPHICS_ON |
1427                                PVRSRV_SGX_HWPERF_MK_EXECUTION_ON)) != 0) {
1428                                 return PVRSRV_ERROR_INVALID_PARAMS;
1429                         }
1430
1431                         pvr_dev_lock();
1432                         ui32MatchingFlags = readl(&psDevInfo->
1433                                                  psSGXHostCtl->ui32HWPerfFlags);
1434                         ui32MatchingFlags &=
1435                                 psMiscInfo->uData.ui32NewHWPerfStatus;
1436                         if ((ui32MatchingFlags & PVRSRV_SGX_HWPERF_GRAPHICS_ON)
1437                             == 0UL) {
1438                                 psHWPerfCB->ui32OrdinalGRAPHICS = 0xffffffff;
1439                         }
1440                         if ((ui32MatchingFlags &
1441                              PVRSRV_SGX_HWPERF_MK_EXECUTION_ON) == 0UL) {
1442                                 psHWPerfCB->ui32OrdinalMK_EXECUTION =
1443                                     0xffffffffUL;
1444                         }
1445
1446
1447                         writel(psMiscInfo->uData.ui32NewHWPerfStatus,
1448                                 &psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1449                         pvr_dev_unlock();
1450 #if defined(PDUMP)
1451                         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
1452                                               "SGX ukernel HWPerf status %u\n",
1453                                               readl(&psDevInfo->psSGXHostCtl->
1454                                                               ui32HWPerfFlags));
1455                         PDUMPMEM(NULL, psDevInfo->psKernelSGXHostCtlMemInfo,
1456                                  offsetof(struct SGXMKIF_HOST_CTL,
1457                                           ui32HWPerfFlags),
1458                                  sizeof(psDevInfo->psSGXHostCtl->
1459                                         ui32HWPerfFlags),
1460                                  PDUMP_FLAGS_CONTINUOUS,
1461                                  MAKEUNIQUETAG(psDevInfo->
1462                                                psKernelSGXHostCtlMemInfo));
1463 #endif
1464
1465                         return PVRSRV_OK;
1466                 }
1467         case SGX_MISC_INFO_REQUEST_HWPERF_CB_ON:
1468                 {
1469
1470                         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1471                             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1472                         u32 l;
1473
1474                         psHWPerfCB->ui32OrdinalGRAPHICS = 0xffffffffUL;
1475
1476                         pvr_dev_lock();;
1477                         l = readl(&psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1478                         l |= PVRSRV_SGX_HWPERF_GRAPHICS_ON;
1479                         writel(l, &psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1480                         pvr_dev_unlock();
1481
1482                         return PVRSRV_OK;
1483                 }
1484         case SGX_MISC_INFO_REQUEST_HWPERF_CB_OFF:
1485                 {
1486                         pvr_dev_lock();
1487                         writel(0, &psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1488                         pvr_dev_unlock();
1489
1490                         return PVRSRV_OK;
1491                 }
1492         case SGX_MISC_INFO_REQUEST_HWPERF_RETRIEVE_CB:
1493                 {
1494                         struct SGX_MISC_INFO_HWPERF_RETRIEVE_CB *psRetrieve =
1495                             &psMiscInfo->uData.sRetrieveCB;
1496                         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1497                             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1498                         unsigned i;
1499
1500                         for (i = 0;
1501                              psHWPerfCB->ui32Woff != psHWPerfCB->ui32Roff
1502                              && i < psRetrieve->ui32ArraySize; i++) {
1503                                 struct SGXMKIF_HWPERF_CB_ENTRY *psData =
1504                                     &psHWPerfCB->psHWPerfCBData[psHWPerfCB->
1505                                                                 ui32Roff];
1506
1507                                 psRetrieve->psHWPerfData[i].ui32FrameNo =
1508                                     psData->ui32FrameNo;
1509                                 psRetrieve->psHWPerfData[i].ui32Type =
1510                                     (psData->ui32Type &
1511                                      PVRSRV_SGX_HWPERF_TYPE_OP_MASK);
1512                                 psRetrieve->psHWPerfData[i].ui32StartTime =
1513                                     psData->ui32Time;
1514                                 psRetrieve->psHWPerfData[i].ui32StartTimeWraps =
1515                                     psData->ui32TimeWraps;
1516                                 psRetrieve->psHWPerfData[i].ui32EndTime =
1517                                     psData->ui32Time;
1518                                 psRetrieve->psHWPerfData[i].ui32EndTimeWraps =
1519                                     psData->ui32TimeWraps;
1520                                 psRetrieve->psHWPerfData[i].ui32ClockSpeed =
1521                                     psDevInfo->ui32CoreClockSpeed;
1522                                 psRetrieve->psHWPerfData[i].ui32TimeMax =
1523                                     psDevInfo->ui32uKernelTimerClock;
1524                                 psHWPerfCB->ui32Roff =
1525                                     (psHWPerfCB->ui32Roff + 1) &
1526                                     (SGXMKIF_HWPERF_CB_SIZE - 1);
1527                         }
1528                         psRetrieve->ui32DataCount = i;
1529                         psRetrieve->ui32Time = OSClockus();
1530                         return PVRSRV_OK;
1531                 }
1532         default:
1533                 {
1534                         return PVRSRV_ERROR_INVALID_PARAMS;
1535                 }
1536         }
1537 }
1538
1539 enum PVRSRV_ERROR SGXReadDiffCountersKM(void *hDevHandle, u32 ui32Reg,
1540                                    u32 *pui32Old, IMG_BOOL bNew, u32 ui32New,
1541                                    u32 ui32NewReset, u32 ui32CountersReg,
1542                                    u32 *pui32Time, IMG_BOOL *pbActive,
1543                                    struct PVRSRV_SGXDEV_DIFF_INFO *psDiffs,
1544                                    int edm_compat_abi)
1545 {
1546         struct SYS_DATA *psSysData;
1547         struct PVRSRV_POWER_DEV *psPowerDevice;
1548         IMG_BOOL bPowered = IMG_FALSE;
1549         struct PVRSRV_DEVICE_NODE *psDeviceNode = hDevHandle;
1550         struct PVRSRV_SGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
1551
1552         if (bNew)
1553                 psDevInfo->ui32HWGroupRequested = ui32New;
1554         psDevInfo->ui32HWReset |= ui32NewReset;
1555
1556         if (SysAcquireData(&psSysData) != PVRSRV_OK)
1557                 return PVRSRV_ERROR_GENERIC;
1558
1559         psPowerDevice = psSysData->psPowerDeviceList;
1560         while (psPowerDevice) {
1561                 if (psPowerDevice->ui32DeviceIndex ==
1562                     psDeviceNode->sDevId.ui32DeviceIndex) {
1563                         bPowered =
1564                             (IMG_BOOL)(psPowerDevice->eCurrentPowerState ==
1565                                         PVRSRV_POWER_STATE_D0);
1566                         break;
1567                 }
1568
1569                 psPowerDevice = psPowerDevice->psNext;
1570         }
1571
1572         *pbActive = bPowered;
1573
1574         pvr_dev_lock();
1575
1576         {
1577                 struct PVRSRV_SGXDEV_DIFF_INFO sNew,
1578                                                *psPrev = &psDevInfo->sDiffInfo;
1579                 u32 i;
1580
1581                 sNew.ui32Time[0] = OSClockus();
1582                 *pui32Time = sNew.ui32Time[0];
1583                 if (sNew.ui32Time[0] != psPrev->ui32Time[0] && bPowered) {
1584                         u32 __iomem *time_wraps;
1585
1586                         *pui32Old =
1587                             OSReadHWReg(psDevInfo->pvRegsBaseKM, ui32Reg);
1588
1589                         for (i = 0; i < PVRSRV_SGX_DIFF_NUM_COUNTERS; ++i) {
1590                                 sNew.aui32Counters[i] =
1591                                     OSReadHWReg(psDevInfo->pvRegsBaseKM,
1592                                                 ui32CountersReg + (i * 4));
1593                         }
1594
1595                         if (psDevInfo->ui32HWGroupRequested != *pui32Old) {
1596                                 if (psDevInfo->ui32HWReset != 0) {
1597                                         OSWriteHWReg(psDevInfo->pvRegsBaseKM,
1598                                                      ui32Reg,
1599                                                      psDevInfo->
1600                                                      ui32HWGroupRequested |
1601                                                      psDevInfo->ui32HWReset);
1602                                         psDevInfo->ui32HWReset = 0;
1603                                 }
1604                                 OSWriteHWReg(psDevInfo->pvRegsBaseKM, ui32Reg,
1605                                              psDevInfo->ui32HWGroupRequested);
1606                         }
1607
1608                         sNew.ui32Marker[0] = psDevInfo->ui32KickTACounter;
1609                         sNew.ui32Marker[1] = psDevInfo->ui32KickTARenderCounter;
1610
1611                         time_wraps = &psDevInfo->psSGXHostCtl->ui32TimeWraps;
1612                         if (edm_compat_abi)
1613                                 time_wraps -= 1;
1614                         sNew.ui32Time[1] = readl(time_wraps);
1615
1616                         for (i = 0; i < PVRSRV_SGX_DIFF_NUM_COUNTERS; ++i) {
1617                                 psDiffs->aui32Counters[i] =
1618                                     sNew.aui32Counters[i] -
1619                                     psPrev->aui32Counters[i];
1620                         }
1621
1622                         psDiffs->ui32Marker[0] =
1623                             sNew.ui32Marker[0] - psPrev->ui32Marker[0];
1624                         psDiffs->ui32Marker[1] =
1625                             sNew.ui32Marker[1] - psPrev->ui32Marker[1];
1626
1627                         psDiffs->ui32Time[0] =
1628                             sNew.ui32Time[0] - psPrev->ui32Time[0];
1629                         psDiffs->ui32Time[1] =
1630                             sNew.ui32Time[1] - psPrev->ui32Time[1];
1631
1632                         *psPrev = sNew;
1633                 } else {
1634                         for (i = 0; i < PVRSRV_SGX_DIFF_NUM_COUNTERS; ++i)
1635                                 psDiffs->aui32Counters[i] = 0;
1636
1637                         psDiffs->ui32Marker[0] = 0;
1638                         psDiffs->ui32Marker[1] = 0;
1639
1640                         psDiffs->ui32Time[0] = 0;
1641                         psDiffs->ui32Time[1] = 0;
1642                 }
1643         }
1644
1645         SGXTestActivePowerEvent(psDeviceNode);
1646
1647         pvr_dev_unlock();
1648
1649         return PVRSRV_OK;
1650 }
1651
1652 enum PVRSRV_ERROR SGXReadHWPerfCBKM(void *hDevHandle, u32 ui32ArraySize,
1653                         struct PVRSRV_SGX_HWPERF_CB_ENTRY *psClientHWPerfEntry,
1654                         u32 *pui32DataCount, u32 *pui32ClockSpeed,
1655                         u32 *pui32HostTimeStamp)
1656 {
1657         enum PVRSRV_ERROR eError = PVRSRV_OK;
1658         struct PVRSRV_DEVICE_NODE *psDeviceNode = hDevHandle;
1659         struct PVRSRV_SGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
1660         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1661             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1662         unsigned i;
1663
1664         for (i = 0;
1665              psHWPerfCB->ui32Woff != psHWPerfCB->ui32Roff && i < ui32ArraySize;
1666              i++) {
1667                 struct SGXMKIF_HWPERF_CB_ENTRY *psMKPerfEntry =
1668                     &psHWPerfCB->psHWPerfCBData[psHWPerfCB->ui32Roff];
1669
1670                 psClientHWPerfEntry[i].ui32FrameNo = psMKPerfEntry->ui32FrameNo;
1671                 psClientHWPerfEntry[i].ui32Type = psMKPerfEntry->ui32Type;
1672                 psClientHWPerfEntry[i].ui32Ordinal = psMKPerfEntry->ui32Ordinal;
1673                 psClientHWPerfEntry[i].ui32Clocksx16 =
1674                     SGXConvertTimeStamp(psDevInfo, psMKPerfEntry->ui32TimeWraps,
1675                                         psMKPerfEntry->ui32Time);
1676                 OSMemCopy(&psClientHWPerfEntry[i].ui32Counters[0],
1677                           &psMKPerfEntry->ui32Counters[0],
1678                           sizeof(psMKPerfEntry->ui32Counters));
1679
1680                 psHWPerfCB->ui32Roff =
1681                     (psHWPerfCB->ui32Roff + 1) & (SGXMKIF_HWPERF_CB_SIZE - 1);
1682         }
1683
1684         *pui32DataCount = i;
1685         *pui32ClockSpeed = psDevInfo->ui32CoreClockSpeed;
1686         *pui32HostTimeStamp = OSClockus();
1687
1688         return eError;
1689 }