pvr/sgxinit.c

   1 /**********************************************************************
   2  *
   3  * Copyright(c) 2008 Imagination Technologies Ltd. All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful but, except
  10  * as otherwise stated in writing, without any warranty; without even the
  11  * implied warranty of merchantability or fitness for a particular purpose.
  12  * See the GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * Imagination Technologies Ltd. <gpl-support@imgtec.com>
  23  * Home Park Estate, Kings Langley, Herts, WD4 8LZ, UK
  24  *
  25  ******************************************************************************/
  26
  27 #include <stddef.h>
  28
  29 #include <linux/workqueue.h>
  30 #include <linux/io.h>
  31 #include <linux/slab.h>
  32 #include <linux/sched.h>
  33 #include <linux/mm.h>
  34 #include <linux/vmalloc.h>
  35
  36 #include "sgxdefs.h"
  37 #include "sgxmmu.h"
  38 #include "services_headers.h"
  39 #include "buffer_manager.h"
  40 #include "sgxapi_km.h"
  41 #include "sgxinfo.h"
  42 #include "sgxinfokm.h"
  43 #include "sgxconfig.h"
  44 #include "sysconfig.h"
  45 #include "pvr_bridge_km.h"
  46 #include "sgx_bridge_km.h"
  47 #include "resman.h"
  48 #include "bridged_support.h"
  49
  50 #include "pvr_pdump.h"
  51 #include "ra.h"
  52 #include "mmu.h"
  53 #include "mm.h"
  54 #include "handle.h"
  55 #include "perproc.h"
  56
  57 #include "sgxutils.h"
  58 #include "pvrversion.h"
  59 #include "sgx_options.h"
  60
  61 #ifdef CONFIG_DEBUG_FS
  62 #include "pvr_debugfs.h"
  63 #endif
  64
  65 static IMG_BOOL SGX_ISRHandler(void *pvData);
  66
  67 static u32 gui32EventStatusServicesByISR;
  68
  69 static enum PVRSRV_ERROR SGXGetBuildInfoKM(struct PVRSRV_SGXDEV_INFO *psDevInfo,
  70                                     struct PVRSRV_DEVICE_NODE *psDeviceNode);
  71
  72 static void SGXCommandComplete(struct PVRSRV_DEVICE_NODE *psDeviceNode)
  73 {
  74         BUG_ON(in_irq());
  75
  76         SGXScheduleProcessQueuesKM(psDeviceNode);
  77 }
  78
  79 static u32 DeinitDevInfo(struct PVRSRV_SGXDEV_INFO *psDevInfo)
  80 {
  81         if (psDevInfo->psKernelCCBInfo != NULL)
  82                 OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP,
  83                           sizeof(struct PVRSRV_SGX_CCB_INFO),
  84                           psDevInfo->psKernelCCBInfo, NULL);
  85
  86         return PVRSRV_OK;
  87 }
  88
  89 static enum PVRSRV_ERROR InitDevInfo(struct PVRSRV_PER_PROCESS_DATA *psPerProc,
  90                                 struct PVRSRV_DEVICE_NODE *psDeviceNode,
  91                                 struct SGX_BRIDGE_INIT_INFO *psInitInfo)
  92 {
  93         struct PVRSRV_SGXDEV_INFO *psDevInfo =
  94             (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
  95         enum PVRSRV_ERROR eError;
  96
  97         struct PVRSRV_SGX_CCB_INFO *psKernelCCBInfo = NULL;
  98
  99         PVR_UNREFERENCED_PARAMETER(psPerProc);
 100         psDevInfo->sScripts = psInitInfo->sScripts;
 101
 102         psDevInfo->psKernelCCBMemInfo =
 103             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelCCBMemInfo;
 104         psDevInfo->psKernelCCB =
 105             (struct PVRSRV_SGX_KERNEL_CCB *)psDevInfo->psKernelCCBMemInfo->
 106                                                     pvLinAddrKM;
 107
 108         psDevInfo->psKernelCCBCtlMemInfo =
 109             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelCCBCtlMemInfo;
 110         psDevInfo->psKernelCCBCtl =
 111             (struct PVRSRV_SGX_CCB_CTL *)psDevInfo->psKernelCCBCtlMemInfo->
 112                                                     pvLinAddrKM;
 113
 114         psDevInfo->psKernelCCBEventKickerMemInfo =
 115             (struct PVRSRV_KERNEL_MEM_INFO *)
 116                             psInitInfo->hKernelCCBEventKickerMemInfo;
 117         psDevInfo->pui32KernelCCBEventKicker =
 118             (u32 *)psDevInfo->psKernelCCBEventKickerMemInfo->pvLinAddrKM;
 119
 120         psDevInfo->psKernelSGXHostCtlMemInfo =
 121             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->
 122                                                     hKernelSGXHostCtlMemInfo;
 123         psDevInfo->psSGXHostCtl = (struct SGXMKIF_HOST_CTL __force __iomem *)
 124                 psDevInfo->psKernelSGXHostCtlMemInfo->pvLinAddrKM;
 125
 126         psDevInfo->psKernelSGXTA3DCtlMemInfo =
 127             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->
 128                                                     hKernelSGXTA3DCtlMemInfo;
 129
 130         psDevInfo->psKernelSGXMiscMemInfo =
 131             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelSGXMiscMemInfo;
 132
 133         psDevInfo->psKernelHWPerfCBMemInfo =
 134             (struct PVRSRV_KERNEL_MEM_INFO *)psInitInfo->hKernelHWPerfCBMemInfo;
 135         psDevInfo->psKernelEDMStatusBufferMemInfo =
 136                                     psInitInfo->hKernelEDMStatusBufferMemInfo;
 137
 138         eError = OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
 139                             sizeof(struct PVRSRV_SGX_CCB_INFO),
 140                             (void **)&psKernelCCBInfo, NULL);
 141         if (eError != PVRSRV_OK) {
 142                 PVR_DPF(PVR_DBG_ERROR, "InitDevInfo: Failed to alloc memory");
 143                 goto failed_allockernelccb;
 144         }
 145
 146         OSMemSet(psKernelCCBInfo, 0, sizeof(struct PVRSRV_SGX_CCB_INFO));
 147         psKernelCCBInfo->psCCBMemInfo = psDevInfo->psKernelCCBMemInfo;
 148         psKernelCCBInfo->psCCBCtlMemInfo = psDevInfo->psKernelCCBCtlMemInfo;
 149         psKernelCCBInfo->psCommands = psDevInfo->psKernelCCB->asCommands;
 150         psKernelCCBInfo->pui32WriteOffset =
 151                                 &psDevInfo->psKernelCCBCtl->ui32WriteOffset;
 152         psKernelCCBInfo->pui32ReadOffset =
 153                                 &psDevInfo->psKernelCCBCtl->ui32ReadOffset;
 154         psDevInfo->psKernelCCBInfo = psKernelCCBInfo;
 155
 156         psDevInfo->ui32HostKickAddress = psInitInfo->ui32HostKickAddress;
 157
 158         psDevInfo->ui32GetMiscInfoAddress = psInitInfo->ui32GetMiscInfoAddress;
 159
 160         psDevInfo->bForcePTOff = IMG_FALSE;
 161
 162         psDevInfo->ui32CacheControl = psInitInfo->ui32CacheControl;
 163
 164         psDevInfo->ui32EDMTaskReg0 = psInitInfo->ui32EDMTaskReg0;
 165         psDevInfo->ui32EDMTaskReg1 = psInitInfo->ui32EDMTaskReg1;
 166         psDevInfo->ui32ClkGateStatusReg = psInitInfo->ui32ClkGateStatusReg;
 167         psDevInfo->ui32ClkGateStatusMask = psInitInfo->ui32ClkGateStatusMask;
 168
 169         OSMemCopy(&psDevInfo->asSGXDevData, &psInitInfo->asInitDevData,
 170                   sizeof(psDevInfo->asSGXDevData));
 171
 172         psDevInfo->state_buf_ofs = psInitInfo->state_buf_ofs;
 173
 174         return PVRSRV_OK;
 175
 176 failed_allockernelccb:
 177         DeinitDevInfo(psDevInfo);
 178
 179         return eError;
 180 }
 181
 182 static enum PVRSRV_ERROR SGXRunScript(struct PVRSRV_SGXDEV_INFO *psDevInfo,
 183                                  union SGX_INIT_COMMAND *psScript,
 184                                  u32 ui32NumInitCommands)
 185 {
 186         u32 ui32PC;
 187         union SGX_INIT_COMMAND *psComm;
 188
 189         for (ui32PC = 0, psComm = psScript;
 190              ui32PC < ui32NumInitCommands; ui32PC++, psComm++) {
 191                 switch (psComm->eOp) {
 192                 case SGX_INIT_OP_WRITE_HW_REG:
 193                         {
 194                                 OSWriteHWReg(psDevInfo->pvRegsBaseKM,
 195                                              psComm->sWriteHWReg.ui32Offset,
 196                                              psComm->sWriteHWReg.ui32Value);
 197                                 PDUMPREG(psComm->sWriteHWReg.ui32Offset,
 198                                          psComm->sWriteHWReg.ui32Value);
 199                                 break;
 200                         }
 201 #if defined(PDUMP)
 202                 case SGX_INIT_OP_PDUMP_HW_REG:
 203                         {
 204                                 PDUMPREG(psComm->sPDumpHWReg.ui32Offset,
 205                                          psComm->sPDumpHWReg.ui32Value);
 206                                 break;
 207                         }
 208 #endif
 209                 case SGX_INIT_OP_HALT:
 210                         {
 211                                 return PVRSRV_OK;
 212                         }
 213                 case SGX_INIT_OP_ILLEGAL:
 214
 215                 default:
 216                         {
 217                                 PVR_DPF(PVR_DBG_ERROR,
 218                                      "SGXRunScript: PC %d: Illegal command: %d",
 219                                       ui32PC, psComm->eOp);
 220                                 return PVRSRV_ERROR_GENERIC;
 221                         }
 222                 }
 223
 224         }
 225
 226         return PVRSRV_ERROR_GENERIC;
 227 }
 228
 229 enum PVRSRV_ERROR SGXInitialise(struct PVRSRV_SGXDEV_INFO *psDevInfo,
 230                                   IMG_BOOL bHardwareRecovery)
 231 {
 232         enum PVRSRV_ERROR eError;
 233
 234         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 235                               "SGX initialisation script part 1\n");
 236         eError =
 237             SGXRunScript(psDevInfo, psDevInfo->sScripts.asInitCommandsPart1,
 238                          SGX_MAX_INIT_COMMANDS);
 239         if (eError != PVRSRV_OK) {
 240                 PVR_DPF(PVR_DBG_ERROR,
 241                          "SGXInitialise: SGXRunScript (part 1) failed (%d)",
 242                          eError);
 243                 return PVRSRV_ERROR_GENERIC;
 244         }
 245         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 246                               "End of SGX initialisation script part 1\n");
 247
 248         SGXReset(psDevInfo, PDUMP_FLAGS_CONTINUOUS);
 249
 250
 251
 252         *psDevInfo->pui32KernelCCBEventKicker = 0;
 253 #if defined(PDUMP)
 254         PDUMPMEM(NULL, psDevInfo->psKernelCCBEventKickerMemInfo, 0,
 255                  sizeof(*psDevInfo->pui32KernelCCBEventKicker),
 256                  PDUMP_FLAGS_CONTINUOUS,
 257                  MAKEUNIQUETAG(psDevInfo->psKernelCCBEventKickerMemInfo));
 258 #endif
 259
 260         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 261                               "SGX initialisation script part 2\n");
 262         eError =
 263             SGXRunScript(psDevInfo, psDevInfo->sScripts.asInitCommandsPart2,
 264                          SGX_MAX_INIT_COMMANDS);
 265         if (eError != PVRSRV_OK) {
 266                 PVR_DPF(PVR_DBG_ERROR,
 267                          "SGXInitialise: SGXRunScript (part 2) failed (%d)",
 268                          eError);
 269                 return PVRSRV_ERROR_GENERIC;
 270         }
 271         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
 272                               "End of SGX initialisation script part 2\n");
 273
 274         SGXStartTimer(psDevInfo, (IMG_BOOL)!bHardwareRecovery);
 275
 276         if (bHardwareRecovery) {
 277                 struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 278                     psDevInfo->psSGXHostCtl;
 279
 280                 if (PollForValueKM(&psSGXHostCtl->ui32InterruptClearFlags, 0,
 281                      PVRSRV_USSE_EDM_INTERRUPT_HWR,
 282                      MAX_HW_TIME_US / WAIT_TRY_COUNT, 1000) != PVRSRV_OK) {
 283                         PVR_DPF(PVR_DBG_ERROR, "SGXInitialise: "
 284                                         "Wait for uKernel HW Recovery failed");
 285                         PVR_DBG_BREAK;
 286                         return PVRSRV_ERROR_RETRY;
 287                 }
 288         }
 289
 290         PVR_ASSERT(psDevInfo->psKernelCCBCtl->ui32ReadOffset ==
 291                    psDevInfo->psKernelCCBCtl->ui32WriteOffset);
 292
 293         return PVRSRV_OK;
 294 }
 295
 296 enum PVRSRV_ERROR SGXDeinitialise(void *hDevCookie)
 297 {
 298         struct PVRSRV_SGXDEV_INFO *psDevInfo = (struct PVRSRV_SGXDEV_INFO *)
 299                                                                    hDevCookie;
 300         enum PVRSRV_ERROR eError;
 301
 302         if (psDevInfo->pvRegsBaseKM == NULL)
 303                 return PVRSRV_OK;
 304
 305         eError = SGXRunScript(psDevInfo, psDevInfo->sScripts.asDeinitCommands,
 306                          SGX_MAX_DEINIT_COMMANDS);
 307         if (eError != PVRSRV_OK) {
 308                 PVR_DPF(PVR_DBG_ERROR,
 309                          "SGXDeinitialise: SGXRunScript failed (%d)", eError);
 310                 return PVRSRV_ERROR_GENERIC;
 311         }
 312
 313         return PVRSRV_OK;
 314 }
 315
 316 static enum PVRSRV_ERROR DevInitSGXPart1(void *pvDeviceNode)
 317 {
 318         struct PVRSRV_SGXDEV_INFO *psDevInfo;
 319         void *hKernelDevMemContext;
 320         struct IMG_DEV_PHYADDR sPDDevPAddr;
 321         u32 i;
 322         struct PVRSRV_DEVICE_NODE *psDeviceNode = (struct PVRSRV_DEVICE_NODE *)
 323                                                                    pvDeviceNode;
 324         struct DEVICE_MEMORY_HEAP_INFO *psDeviceMemoryHeap =
 325             psDeviceNode->sDevMemoryInfo.psDeviceMemoryHeap;
 326         enum PVRSRV_ERROR eError;
 327
 328         PDUMPCOMMENT("SGX Initialisation Part 1");
 329
 330         PDUMPCOMMENT("SGX Core Version Information: %s",
 331                      SGX_CORE_FRIENDLY_NAME);
 332         PDUMPCOMMENT("SGX Core Revision Information: multi rev support");
 333
 334         if (OSAllocMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 335                        sizeof(struct PVRSRV_SGXDEV_INFO),
 336                        (void **)&psDevInfo, NULL) != PVRSRV_OK) {
 337                 PVR_DPF(PVR_DBG_ERROR,
 338                         "DevInitSGXPart1 : Failed to alloc memory for DevInfo");
 339                 return PVRSRV_ERROR_OUT_OF_MEMORY;
 340         }
 341         OSMemSet(psDevInfo, 0, sizeof(struct PVRSRV_SGXDEV_INFO));
 342
 343         psDevInfo->eDeviceType = DEV_DEVICE_TYPE;
 344         psDevInfo->eDeviceClass = DEV_DEVICE_CLASS;
 345
 346         psDeviceNode->pvDevice = (void *) psDevInfo;
 347
 348         psDevInfo->pvDeviceMemoryHeap = (void *) psDeviceMemoryHeap;
 349
 350         hKernelDevMemContext = BM_CreateContext(psDeviceNode, &sPDDevPAddr,
 351                                                 NULL, NULL);
 352         if (!hKernelDevMemContext)
 353                 goto err1;
 354
 355         psDevInfo->sKernelPDDevPAddr = sPDDevPAddr;
 356
 357         for (i = 0; i < psDeviceNode->sDevMemoryInfo.ui32HeapCount; i++) {
 358                 void *hDevMemHeap;
 359
 360                 switch (psDeviceMemoryHeap[i].DevMemHeapType) {
 361                 case DEVICE_MEMORY_HEAP_KERNEL:
 362                 case DEVICE_MEMORY_HEAP_SHARED:
 363                 case DEVICE_MEMORY_HEAP_SHARED_EXPORTED:
 364                         {
 365                                 hDevMemHeap =
 366                                     BM_CreateHeap(hKernelDevMemContext,
 367                                                   &psDeviceMemoryHeap[i]);
 368
 369                                 if (!hDevMemHeap)
 370                                         goto err2;
 371
 372                                 psDeviceMemoryHeap[i].hDevMemHeap = hDevMemHeap;
 373                                 break;
 374                         }
 375                 }
 376         }
 377
 378         eError = MMU_BIFResetPDAlloc(psDevInfo);
 379         if (eError != PVRSRV_OK) {
 380                 PVR_DPF(PVR_DBG_ERROR,
 381                          "DevInitSGX : Failed to alloc memory for BIF reset");
 382                 goto err2;
 383         }
 384
 385         return PVRSRV_OK;
 386 err2:
 387         while (i) {
 388                 int type;
 389
 390                 i--;
 391                 type = psDeviceMemoryHeap[i].DevMemHeapType;
 392                 if (type != DEVICE_MEMORY_HEAP_KERNEL &&
 393                     type != DEVICE_MEMORY_HEAP_SHARED &&
 394                     type != DEVICE_MEMORY_HEAP_SHARED_EXPORTED)
 395                         continue;
 396                 BM_DestroyHeap(psDeviceMemoryHeap[i].hDevMemHeap);
 397         }
 398         BM_DestroyContext(hKernelDevMemContext);
 399 err1:
 400         OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 401                   sizeof(struct PVRSRV_SGXDEV_INFO), psDevInfo, NULL);
 402
 403         return PVRSRV_ERROR_GENERIC;
 404 }
 405
 406 enum PVRSRV_ERROR SGXGetInfoForSrvinitKM(void *hDevHandle,
 407                                 struct SGX_BRIDGE_INFO_FOR_SRVINIT *psInitInfo)
 408 {
 409         struct PVRSRV_DEVICE_NODE *psDeviceNode;
 410         struct PVRSRV_SGXDEV_INFO *psDevInfo;
 411         enum PVRSRV_ERROR eError;
 412
 413         PDUMPCOMMENT("SGXGetInfoForSrvinit");
 414
 415         psDeviceNode = (struct PVRSRV_DEVICE_NODE *)hDevHandle;
 416         psDevInfo = (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 417
 418         psInitInfo->sPDDevPAddr = psDevInfo->sKernelPDDevPAddr;
 419
 420         eError =
 421             PVRSRVGetDeviceMemHeapsKM(hDevHandle, &psInitInfo->asHeapInfo[0]);
 422         if (eError != PVRSRV_OK) {
 423                 PVR_DPF(PVR_DBG_ERROR, "SGXGetInfoForSrvinit: "
 424                                 "PVRSRVGetDeviceMemHeapsKM failed (%d)",
 425                          eError);
 426                 return PVRSRV_ERROR_GENERIC;
 427         }
 428
 429         return eError;
 430 }
 431
 432 enum PVRSRV_ERROR DevInitSGXPart2KM(struct PVRSRV_PER_PROCESS_DATA *psPerProc,
 433                                    void *hDevHandle,
 434                                    struct SGX_BRIDGE_INIT_INFO *psInitInfo)
 435 {
 436         struct PVRSRV_DEVICE_NODE *psDeviceNode;
 437         struct PVRSRV_SGXDEV_INFO *psDevInfo;
 438         enum PVRSRV_ERROR eError;
 439         struct SGX_DEVICE_MAP *psSGXDeviceMap;
 440         enum PVR_POWER_STATE eDefaultPowerState;
 441         u32 l;
 442
 443         PDUMPCOMMENT("SGX Initialisation Part 2");
 444
 445         psDeviceNode = (struct PVRSRV_DEVICE_NODE *)hDevHandle;
 446         psDevInfo = (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 447
 448         eError = InitDevInfo(psPerProc, psDeviceNode, psInitInfo);
 449         if (eError != PVRSRV_OK) {
 450                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM: "
 451                                         "Failed to load EDM program");
 452                 goto failed_init_dev_info;
 453         }
 454
 455
 456         eError = SysGetDeviceMemoryMap(PVRSRV_DEVICE_TYPE_SGX,
 457                                        (void **) &psSGXDeviceMap);
 458         if (eError != PVRSRV_OK) {
 459                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM: "
 460                                         "Failed to get device memory map!");
 461                 return PVRSRV_ERROR_INIT_FAILURE;
 462         }
 463
 464         if (psSGXDeviceMap->pvRegsCpuVBase) {
 465                 psDevInfo->pvRegsBaseKM = psSGXDeviceMap->pvRegsCpuVBase;
 466         } else {
 467                 psDevInfo->pvRegsBaseKM =
 468                     OSMapPhysToLin(psSGXDeviceMap->sRegsCpuPBase,
 469                                    psSGXDeviceMap->ui32RegsSize,
 470                                    PVRSRV_HAP_KERNEL_ONLY | PVRSRV_HAP_UNCACHED,
 471                                    NULL);
 472                 if (!psDevInfo->pvRegsBaseKM) {
 473                         PVR_DPF(PVR_DBG_ERROR,
 474                                  "DevInitSGXPart2KM: Failed to map in regs\n");
 475                         return PVRSRV_ERROR_BAD_MAPPING;
 476                 }
 477         }
 478         psDevInfo->ui32RegSize = psSGXDeviceMap->ui32RegsSize;
 479         psDevInfo->sRegsPhysBase = psSGXDeviceMap->sRegsSysPBase;
 480
 481         psDeviceNode->pvISRData = psDeviceNode;
 482
 483         PVR_ASSERT(psDeviceNode->pfnDeviceISR == SGX_ISRHandler);
 484
 485         pvr_dev_lock();
 486         l = readl(&psDevInfo->psSGXHostCtl->ui32PowerStatus);
 487         l |= PVRSRV_USSE_EDM_POWMAN_NO_WORK;
 488         writel(l, &psDevInfo->psSGXHostCtl->ui32PowerStatus);
 489         pvr_dev_unlock();
 490         eDefaultPowerState = PVRSRV_POWER_STATE_D3;
 491
 492         eError = PVRSRVRegisterPowerDevice(psDeviceNode->sDevId.ui32DeviceIndex,
 493                                            SGXPrePowerStateExt,
 494                                            SGXPostPowerStateExt,
 495                                            SGXPreClockSpeedChange,
 496                                            SGXPostClockSpeedChange,
 497                                            (void *) psDeviceNode,
 498                                            PVRSRV_POWER_STATE_D3,
 499                                            eDefaultPowerState);
 500         if (eError != PVRSRV_OK) {
 501                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM: "
 502                                 "failed to register device with power manager");
 503                 return eError;
 504         }
 505
 506         OSMemSet(psDevInfo->psKernelCCB, 0,
 507                  sizeof(struct PVRSRV_SGX_KERNEL_CCB));
 508         OSMemSet(psDevInfo->psKernelCCBCtl, 0,
 509                  sizeof(struct PVRSRV_SGX_CCB_CTL));
 510         OSMemSet(psDevInfo->pui32KernelCCBEventKicker, 0,
 511                  sizeof(*psDevInfo->pui32KernelCCBEventKicker));
 512         PDUMPCOMMENT("Initialise Kernel CCB");
 513         PDUMPMEM(NULL, psDevInfo->psKernelCCBMemInfo, 0,
 514                  sizeof(struct PVRSRV_SGX_KERNEL_CCB), PDUMP_FLAGS_CONTINUOUS,
 515                  MAKEUNIQUETAG(psDevInfo->psKernelCCBMemInfo));
 516         PDUMPCOMMENT("Initialise Kernel CCB Control");
 517         PDUMPMEM(NULL, psDevInfo->psKernelCCBCtlMemInfo, 0,
 518                  sizeof(struct PVRSRV_SGX_CCB_CTL), PDUMP_FLAGS_CONTINUOUS,
 519                  MAKEUNIQUETAG(psDevInfo->psKernelCCBCtlMemInfo));
 520         PDUMPCOMMENT("Initialise Kernel CCB Event Kicker");
 521         PDUMPMEM(NULL, psDevInfo->psKernelCCBEventKickerMemInfo, 0,
 522                  sizeof(*psDevInfo->pui32KernelCCBEventKicker),
 523                  PDUMP_FLAGS_CONTINUOUS,
 524                  MAKEUNIQUETAG(psDevInfo->psKernelCCBEventKickerMemInfo));
 525
 526         psDevInfo->hTimer = SGXOSTimerInit(psDeviceNode);
 527         if (!psDevInfo->hTimer)
 528                 PVR_DPF(PVR_DBG_ERROR, "DevInitSGXPart2KM : "
 529                         "Failed to initialize HW recovery timer");
 530
 531         return PVRSRV_OK;
 532
 533 failed_init_dev_info:
 534         return eError;
 535 }
 536
 537 static enum PVRSRV_ERROR DevDeInitSGX(void *pvDeviceNode)
 538 {
 539         struct PVRSRV_DEVICE_NODE *psDeviceNode =
 540                 (struct PVRSRV_DEVICE_NODE *)pvDeviceNode;
 541         struct PVRSRV_SGXDEV_INFO *psDevInfo =
 542                 (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 543         enum PVRSRV_ERROR eError;
 544         u32 ui32Heap;
 545         struct DEVICE_MEMORY_HEAP_INFO *psDeviceMemoryHeap;
 546         struct SGX_DEVICE_MAP *psSGXDeviceMap;
 547
 548         if (!psDevInfo) {
 549                 PVR_DPF(PVR_DBG_ERROR, "DevDeInitSGX: Null DevInfo");
 550                 return PVRSRV_OK;
 551         }
 552         if (psDevInfo->hTimer) {
 553                 SGXOSTimerCancel(psDevInfo->hTimer);
 554                 SGXOSTimerDeInit(psDevInfo->hTimer);
 555                 psDevInfo->hTimer = NULL;
 556         }
 557
 558         MMU_BIFResetPDFree(psDevInfo);
 559
 560         DeinitDevInfo(psDevInfo);
 561
 562
 563         psDeviceMemoryHeap =
 564             (struct DEVICE_MEMORY_HEAP_INFO *)psDevInfo->pvDeviceMemoryHeap;
 565         for (ui32Heap = 0;
 566              ui32Heap < psDeviceNode->sDevMemoryInfo.ui32HeapCount;
 567              ui32Heap++) {
 568                 switch (psDeviceMemoryHeap[ui32Heap].DevMemHeapType) {
 569                 case DEVICE_MEMORY_HEAP_KERNEL:
 570                 case DEVICE_MEMORY_HEAP_SHARED:
 571                 case DEVICE_MEMORY_HEAP_SHARED_EXPORTED:
 572                         {
 573                                 if (psDeviceMemoryHeap[ui32Heap].hDevMemHeap !=
 574                                     NULL)
 575                                         BM_DestroyHeap(psDeviceMemoryHeap
 576                                                        [ui32Heap].hDevMemHeap);
 577                                 break;
 578                         }
 579                 }
 580         }
 581
 582         if (!pvr_put_ctx(psDeviceNode->sDevMemoryInfo.pBMKernelContext))
 583                 pr_err("%s: kernel context still in use, can't free it",
 584                         __func__);
 585
 586         eError = PVRSRVRemovePowerDevice(
 587                                 ((struct PVRSRV_DEVICE_NODE *)pvDeviceNode)->
 588                                   sDevId.ui32DeviceIndex);
 589         if (eError != PVRSRV_OK)
 590                 return eError;
 591
 592         eError = SysGetDeviceMemoryMap(PVRSRV_DEVICE_TYPE_SGX,
 593                                        (void **)&psSGXDeviceMap);
 594         if (eError != PVRSRV_OK) {
 595                 PVR_DPF(PVR_DBG_ERROR,
 596                          "DevDeInitSGX: Failed to get device memory map!");
 597                 return eError;
 598         }
 599
 600         if (!psSGXDeviceMap->pvRegsCpuVBase)
 601                 if (psDevInfo->pvRegsBaseKM != NULL)
 602                         OSUnMapPhysToLin(psDevInfo->pvRegsBaseKM,
 603                                          psDevInfo->ui32RegSize,
 604                                          PVRSRV_HAP_KERNEL_ONLY |
 605                                                  PVRSRV_HAP_UNCACHED,
 606                                          NULL);
 607
 608         OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 609                   sizeof(struct PVRSRV_SGXDEV_INFO), psDevInfo, NULL);
 610
 611         psDeviceNode->pvDevice = NULL;
 612
 613         if (psDeviceMemoryHeap != NULL)
 614                 OSFreeMem(PVRSRV_OS_NON_PAGEABLE_HEAP,
 615                           sizeof(struct DEVICE_MEMORY_HEAP_INFO) *
 616                           psDeviceNode->sDevMemoryInfo.ui32HeapCount,
 617                           psDeviceMemoryHeap, NULL);
 618
 619         return PVRSRV_OK;
 620 }
 621
 622 static struct PVRSRV_PER_PROCESS_DATA *find_cur_proc_data(
 623                                         struct PVRSRV_DEVICE_NODE *dev)
 624 {
 625         struct PVRSRV_SGXDEV_INFO *dev_info = dev->pvDevice;
 626         u32 page_dir = readl(dev_info->pvRegsBaseKM +
 627                                 EUR_CR_BIF_DIR_LIST_BASE0);
 628         struct BM_CONTEXT *bm_ctx;
 629         struct RESMAN_CONTEXT *res_ctx = NULL;
 630         struct PVRSRV_PER_PROCESS_DATA *proc_data = NULL;
 631
 632         bm_ctx = bm_find_context(dev->sDevMemoryInfo.pBMContext, page_dir);
 633         if (bm_ctx)
 634                 res_ctx = pvr_get_resman_ctx(bm_ctx);
 635
 636         if (res_ctx)
 637                 proc_data = pvr_get_proc_by_ctx(res_ctx);
 638
 639         return proc_data;
 640 }
 641
 642 static void pr_err_process_info(struct PVRSRV_PER_PROCESS_DATA *proc)
 643 {
 644         struct task_struct *tsk;
 645         int pid;
 646
 647         if (!proc)
 648                 return;
 649
 650         pid = proc->ui32PID;
 651         rcu_read_lock();
 652         tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
 653         pr_err("PID = %d, process name = %s\n", pid, tsk->comm);
 654         rcu_read_unlock();
 655 }
 656
 657 static void pr_err_sgx_registers(struct PVRSRV_SGXDEV_INFO *psDevInfo)
 658 {
 659         pr_err("EVENT_STATUS =     0x%08X\n"
 660                 "EVENT_STATUS2 =    0x%08X\n"
 661                 "BIF_CTRL =         0x%08X\n"
 662                 "BIF_INT_STAT =     0x%08X\n"
 663                 "BIF_MEM_REQ_STAT = 0x%08X\n"
 664                 "BIF_FAULT  =       0x%08X\n"
 665                 "CLKGATECTL =       0x%08X\n",
 666                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_EVENT_STATUS),
 667                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_EVENT_STATUS2),
 668                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_CTRL),
 669                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_INT_STAT),
 670                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_MEM_REQ_STAT),
 671                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_BIF_FAULT),
 672                 readl(psDevInfo->pvRegsBaseKM + EUR_CR_CLKGATECTL));
 673 }
 674
 675 /* Should be called with pvr_lock held */
 676 void
 677 HWRecoveryResetSGX(struct PVRSRV_DEVICE_NODE *psDeviceNode, const char *caller)
 678 {
 679         enum PVRSRV_ERROR eError;
 680         struct PVRSRV_SGXDEV_INFO *psDevInfo =
 681             (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 682         struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 683                                         psDevInfo->psSGXHostCtl;
 684         struct PVRSRV_PER_PROCESS_DATA *proc;
 685         u32 l;
 686         int max_retries = 10;
 687
 688         BUG_ON(!pvr_is_locked());
 689
 690         l = readl(&psSGXHostCtl->ui32InterruptClearFlags);
 691         l |= PVRSRV_USSE_EDM_INTERRUPT_HWR;
 692         writel(l, &psSGXHostCtl->ui32InterruptClearFlags);
 693
 694         pr_err("SGX Hardware Recovery triggered (from %s)\n", caller);
 695
 696         proc = find_cur_proc_data(psDeviceNode);
 697
 698         pr_err_process_info(proc);
 699         pr_err_sgx_registers(psDevInfo);
 700 #ifdef PVRSRV_USSE_EDM_STATUS_DEBUG
 701         edm_trace_print(psDevInfo, NULL, 0);
 702 #endif
 703
 704 #ifdef CONFIG_DEBUG_FS
 705         pvr_hwrec_dump(proc, psDevInfo);
 706 #endif
 707
 708         PDUMPSUSPEND();
 709
 710         do {
 711                 eError = SGXInitialise(psDevInfo, IMG_TRUE);
 712                 if (eError != PVRSRV_ERROR_RETRY)
 713                         break;
 714         } while (max_retries--);
 715
 716         if (eError != PVRSRV_OK) {
 717                 pr_err("%s: recovery failed (%d). Disabling the driver",
 718                         __func__, eError);
 719                 pvr_disable();
 720
 721                 PDUMPRESUME();
 722
 723                 return;
 724         }
 725
 726         PDUMPRESUME();
 727
 728         SGXScheduleProcessQueues(psDeviceNode);
 729
 730         PVRSRVProcessQueues(IMG_TRUE);
 731 }
 732
 733 static unsigned long sgx_reset_forced;
 734
 735 static void SGXOSTimer(struct work_struct *work)
 736 {
 737         struct timer_work_data *data = container_of(work,
 738                                                     struct timer_work_data,
 739                                                     work.work);
 740         struct PVRSRV_DEVICE_NODE *psDeviceNode = data->psDeviceNode;
 741         struct PVRSRV_SGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
 742         static u32 ui32EDMTasks;
 743         static u32 ui32LockupCounter;
 744         static u32 ui32NumResets;
 745         u32 ui32CurrentEDMTasks;
 746         IMG_BOOL bLockup = IMG_FALSE;
 747         IMG_BOOL bPoweredDown;
 748
 749         pvr_lock();
 750
 751         if (!data->armed || pvr_is_disabled()) {
 752                 pvr_unlock();
 753                 return;
 754         }
 755
 756         psDevInfo->ui32TimeStamp++;
 757
 758 #if defined(NO_HARDWARE)
 759         bPoweredDown = IMG_TRUE;
 760 #else
 761         bPoweredDown = (IMG_BOOL) !SGXIsDevicePowered(psDeviceNode);
 762 #endif
 763
 764         if (bPoweredDown) {
 765                 ui32LockupCounter = 0;
 766         } else {
 767                 pvr_dev_lock();
 768                 ui32CurrentEDMTasks = OSReadHWReg(psDevInfo->pvRegsBaseKM,
 769                                                 psDevInfo->ui32EDMTaskReg0);
 770                 if (psDevInfo->ui32EDMTaskReg1 != 0)
 771                         ui32CurrentEDMTasks ^=
 772                             OSReadHWReg(psDevInfo->pvRegsBaseKM,
 773                                         psDevInfo->ui32EDMTaskReg1);
 774                 if ((ui32CurrentEDMTasks == ui32EDMTasks) &&
 775                     (psDevInfo->ui32NumResets == ui32NumResets)) {
 776                         ui32LockupCounter++;
 777                         if (ui32LockupCounter == 3) {
 778                                 ui32LockupCounter = 0;
 779                                 PVR_DPF(PVR_DBG_ERROR, "SGXOSTimer() "
 780                                         "detected SGX lockup (0x%x tasks)",
 781                                          ui32EDMTasks);
 782
 783                                 bLockup = IMG_TRUE;
 784                         }
 785                 } else {
 786                         ui32LockupCounter = 0;
 787                         ui32EDMTasks = ui32CurrentEDMTasks;
 788                         ui32NumResets = psDevInfo->ui32NumResets;
 789                 }
 790                 pvr_dev_unlock();
 791         }
 792
 793         bLockup |= cmpxchg(&sgx_reset_forced, 1, 0);
 794
 795         if (bLockup) {
 796                 struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 797                                                 psDevInfo->psSGXHostCtl;
 798                 u32 l;
 799
 800                 pvr_dev_lock();
 801                 l = readl(&psSGXHostCtl->ui32HostDetectedLockups);
 802                 l++;
 803                 writel(l, &psSGXHostCtl->ui32HostDetectedLockups);
 804
 805                 HWRecoveryResetSGX(psDeviceNode, __func__);
 806                 pvr_dev_unlock();
 807         }
 808
 809         queue_delayed_work(data->work_queue, &data->work,
 810                            msecs_to_jiffies(data->interval));
 811
 812         pvr_unlock();
 813 }
 814
 815 struct timer_work_data *
 816 SGXOSTimerInit(struct PVRSRV_DEVICE_NODE *psDeviceNode)
 817 {
 818         struct timer_work_data *data;
 819
 820         data = kzalloc(sizeof(struct timer_work_data), GFP_KERNEL);
 821         if (!data)
 822                 return NULL;
 823
 824         data->work_queue = create_workqueue("SGXOSTimer");
 825         if (!data->work_queue) {
 826                 kfree(data);
 827                 return NULL;
 828         }
 829
 830         data->interval = 150;
 831         data->psDeviceNode = psDeviceNode;
 832         INIT_DELAYED_WORK(&data->work, SGXOSTimer);
 833
 834         return data;
 835 }
 836
 837 void SGXOSTimerDeInit(struct timer_work_data *data)
 838 {
 839         data->armed = false;
 840         destroy_workqueue(data->work_queue);
 841         kfree(data);
 842 }
 843
 844 enum PVRSRV_ERROR SGXOSTimerEnable(struct timer_work_data *data)
 845 {
 846         if (!data)
 847                 return PVRSRV_ERROR_GENERIC;
 848
 849         if (queue_delayed_work(data->work_queue, &data->work,
 850                                msecs_to_jiffies(data->interval))) {
 851                 data->armed = true;
 852                 return PVRSRV_OK;
 853         }
 854
 855         return PVRSRV_ERROR_GENERIC;
 856 }
 857
 858 enum PVRSRV_ERROR SGXOSTimerCancel(struct timer_work_data *data)
 859 {
 860         if (!data)
 861                 return PVRSRV_ERROR_GENERIC;
 862
 863         data->armed = false;
 864         cancel_delayed_work(&data->work);
 865
 866         return PVRSRV_OK;
 867 }
 868
 869 int sgx_force_reset(void)
 870 {
 871         return !cmpxchg(&sgx_reset_forced, 0, 1);
 872 }
 873
 874 static IMG_BOOL SGX_ISRHandler(void *pvData)
 875 {
 876         IMG_BOOL bInterruptProcessed = IMG_FALSE;
 877
 878         {
 879                 u32 ui32EventStatus, ui32EventEnable;
 880                 u32 ui32EventClear = 0;
 881                 struct PVRSRV_DEVICE_NODE *psDeviceNode;
 882                 struct PVRSRV_SGXDEV_INFO *psDevInfo;
 883
 884                 if (pvData == NULL) {
 885                         PVR_DPF(PVR_DBG_ERROR,
 886                                  "SGX_ISRHandler: Invalid params\n");
 887                         return bInterruptProcessed;
 888                 }
 889
 890                 psDeviceNode = (struct PVRSRV_DEVICE_NODE *)pvData;
 891                 psDevInfo = (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 892
 893                 ui32EventStatus =
 894                     OSReadHWReg(psDevInfo->pvRegsBaseKM, EUR_CR_EVENT_STATUS);
 895                 ui32EventEnable = OSReadHWReg(psDevInfo->pvRegsBaseKM,
 896                                                 EUR_CR_EVENT_HOST_ENABLE);
 897
 898                 gui32EventStatusServicesByISR = ui32EventStatus;
 899
 900                 ui32EventStatus &= ui32EventEnable;
 901
 902                 if (ui32EventStatus & EUR_CR_EVENT_STATUS_SW_EVENT_MASK)
 903                         ui32EventClear |= EUR_CR_EVENT_HOST_CLEAR_SW_EVENT_MASK;
 904
 905                 if (ui32EventClear) {
 906                         bInterruptProcessed = IMG_TRUE;
 907
 908                         ui32EventClear |=
 909                             EUR_CR_EVENT_HOST_CLEAR_MASTER_INTERRUPT_MASK;
 910
 911                         OSWriteHWReg(psDevInfo->pvRegsBaseKM,
 912                                      EUR_CR_EVENT_HOST_CLEAR, ui32EventClear);
 913                 }
 914         }
 915
 916         return bInterruptProcessed;
 917 }
 918
 919 static void SGX_MISRHandler(void *pvData)
 920 {
 921         struct PVRSRV_DEVICE_NODE *psDeviceNode =
 922                         (struct PVRSRV_DEVICE_NODE *)pvData;
 923         struct PVRSRV_SGXDEV_INFO *psDevInfo =
 924                         (struct PVRSRV_SGXDEV_INFO *)psDeviceNode->pvDevice;
 925         struct SGXMKIF_HOST_CTL __iomem *psSGXHostCtl =
 926                         psDevInfo->psSGXHostCtl;
 927         u32 l1, l2;
 928         int dev_idx;
 929         enum PVRSRV_ERROR err;
 930
 931         dev_idx = psDeviceNode->sDevId.ui32DeviceIndex;
 932
 933         pvr_dev_lock();
 934
 935         err = PVRSRVSetDevicePowerStateKM(dev_idx, PVRSRV_POWER_STATE_D0);
 936         BUG_ON(err != PVRSRV_OK);
 937
 938         l1 = readl(&psSGXHostCtl->ui32InterruptFlags);
 939         l2 = readl(&psSGXHostCtl->ui32InterruptClearFlags);
 940
 941         if ((l1 & PVRSRV_USSE_EDM_INTERRUPT_HWR) &&
 942             !(l2 & PVRSRV_USSE_EDM_INTERRUPT_HWR))
 943                 HWRecoveryResetSGX(psDeviceNode, __func__);
 944
 945         if (psDeviceNode->bReProcessDeviceCommandComplete)
 946                 SGXScheduleProcessQueues(psDeviceNode);
 947
 948         SGXTestActivePowerEvent(psDeviceNode);
 949
 950         pvr_dev_unlock();
 951 }
 952
 953 enum PVRSRV_ERROR SGXRegisterDevice(struct PVRSRV_DEVICE_NODE *psDeviceNode)
 954 {
 955         struct DEVICE_MEMORY_INFO *psDevMemoryInfo;
 956         struct DEVICE_MEMORY_HEAP_INFO *psDeviceMemoryHeap;
 957
 958         psDeviceNode->sDevId.eDeviceType = DEV_DEVICE_TYPE;
 959         psDeviceNode->sDevId.eDeviceClass = DEV_DEVICE_CLASS;
 960
 961         psDeviceNode->pfnInitDevice = DevInitSGXPart1;
 962         psDeviceNode->pfnDeInitDevice = DevDeInitSGX;
 963
 964         psDeviceNode->pfnInitDeviceCompatCheck = SGXDevInitCompatCheck;
 965
 966         psDeviceNode->pfnMMUInitialise = MMU_Initialise;
 967         psDeviceNode->pfnMMUFinalise = MMU_Finalise;
 968         psDeviceNode->pfnMMUInsertHeap = MMU_InsertHeap;
 969         psDeviceNode->pfnMMUCreate = MMU_Create;
 970         psDeviceNode->pfnMMUDelete = MMU_Delete;
 971         psDeviceNode->pfnMMUAlloc = MMU_Alloc;
 972         psDeviceNode->pfnMMUFree = MMU_Free;
 973         psDeviceNode->pfnMMUMapPages = MMU_MapPages;
 974         psDeviceNode->pfnMMUMapShadow = MMU_MapShadow;
 975         psDeviceNode->pfnMMUUnmapPages = MMU_UnmapPages;
 976         psDeviceNode->pfnMMUMapScatter = MMU_MapScatter;
 977         psDeviceNode->pfnMMUGetPhysPageAddr = MMU_GetPhysPageAddr;
 978         psDeviceNode->pfnMMUGetPDDevPAddr = MMU_GetPDDevPAddr;
 979
 980         psDeviceNode->pfnDeviceISR = SGX_ISRHandler;
 981         psDeviceNode->pfnDeviceMISR = SGX_MISRHandler;
 982
 983         psDeviceNode->pfnDeviceCommandComplete = SGXCommandComplete;
 984
 985         psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
 986
 987         psDevMemoryInfo->ui32AddressSpaceSizeLog2 =
 988             SGX_FEATURE_ADDRESS_SPACE_SIZE;
 989
 990         psDevMemoryInfo->ui32Flags = 0;
 991         psDevMemoryInfo->ui32HeapCount = SGX_MAX_HEAP_ID;
 992         psDevMemoryInfo->ui32SyncHeapID = SGX_SYNCINFO_HEAP_ID;
 993
 994         psDevMemoryInfo->ui32MappingHeapID = SGX_GENERAL_HEAP_ID;
 995
 996         if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
 997                        sizeof(struct DEVICE_MEMORY_HEAP_INFO) *
 998                        psDevMemoryInfo->ui32HeapCount,
 999                        (void **) &psDevMemoryInfo->psDeviceMemoryHeap,
1000                        NULL) != PVRSRV_OK) {
1001                 PVR_DPF(PVR_DBG_ERROR, "SGXRegisterDevice : "
1002                                 "Failed to alloc memory for "
1003                                 "struct DEVICE_MEMORY_HEAP_INFO");
1004                 return PVRSRV_ERROR_OUT_OF_MEMORY;
1005         }
1006         OSMemSet(psDevMemoryInfo->psDeviceMemoryHeap, 0,
1007                  sizeof(struct DEVICE_MEMORY_HEAP_INFO) *
1008                  psDevMemoryInfo->ui32HeapCount);
1009
1010         psDeviceMemoryHeap = psDevMemoryInfo->psDeviceMemoryHeap;
1011
1012         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32HeapID =
1013             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_GENERAL_HEAP_ID);
1014         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].sDevVAddrBase.uiAddr =
1015             SGX_GENERAL_HEAP_BASE;
1016         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32HeapSize =
1017             SGX_GENERAL_HEAP_SIZE;
1018         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32Attribs =
1019             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1020             PVRSRV_HAP_SINGLE_PROCESS;
1021         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].pszName = "General";
1022         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].pszBSName = "General BS";
1023         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].DevMemHeapType =
1024             DEVICE_MEMORY_HEAP_PERCONTEXT;
1025
1026         psDeviceMemoryHeap[SGX_GENERAL_HEAP_ID].ui32DataPageSize =
1027             SGX_MMU_PAGE_SIZE;
1028
1029         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32HeapID =
1030             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_TADATA_HEAP_ID);
1031         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].sDevVAddrBase.uiAddr =
1032             SGX_TADATA_HEAP_BASE;
1033         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32HeapSize =
1034             SGX_TADATA_HEAP_SIZE;
1035         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32Attribs =
1036             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION
1037             | PVRSRV_HAP_MULTI_PROCESS;
1038         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].pszName = "TA Data";
1039         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].pszBSName = "TA Data BS";
1040         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].DevMemHeapType =
1041             DEVICE_MEMORY_HEAP_PERCONTEXT;
1042
1043         psDeviceMemoryHeap[SGX_TADATA_HEAP_ID].ui32DataPageSize =
1044             SGX_MMU_PAGE_SIZE;
1045
1046         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32HeapID =
1047             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_KERNEL_CODE_HEAP_ID);
1048         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].sDevVAddrBase.uiAddr =
1049             SGX_KERNEL_CODE_HEAP_BASE;
1050         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32HeapSize =
1051             SGX_KERNEL_CODE_HEAP_SIZE;
1052         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32Attribs =
1053             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1054             PVRSRV_HAP_MULTI_PROCESS;
1055         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].pszName = "Kernel Code";
1056         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].pszBSName =
1057             "Kernel Code BS";
1058         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].DevMemHeapType =
1059             DEVICE_MEMORY_HEAP_SHARED_EXPORTED;
1060
1061         psDeviceMemoryHeap[SGX_KERNEL_CODE_HEAP_ID].ui32DataPageSize =
1062             SGX_MMU_PAGE_SIZE;
1063
1064         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32HeapID =
1065             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_KERNEL_DATA_HEAP_ID);
1066         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].sDevVAddrBase.uiAddr =
1067             SGX_KERNEL_DATA_HEAP_BASE;
1068         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32HeapSize =
1069             SGX_KERNEL_DATA_HEAP_SIZE;
1070         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32Attribs =
1071             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1072             PVRSRV_HAP_MULTI_PROCESS;
1073         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].pszName = "KernelData";
1074         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].pszBSName = "KernelData BS";
1075         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].DevMemHeapType =
1076             DEVICE_MEMORY_HEAP_SHARED_EXPORTED;
1077
1078         psDeviceMemoryHeap[SGX_KERNEL_DATA_HEAP_ID].ui32DataPageSize =
1079             SGX_MMU_PAGE_SIZE;
1080
1081         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32HeapID =
1082             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_PIXELSHADER_HEAP_ID);
1083         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].sDevVAddrBase.uiAddr =
1084             SGX_PIXELSHADER_HEAP_BASE;
1085         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32HeapSize =
1086             SGX_PIXELSHADER_HEAP_SIZE;
1087         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32Attribs =
1088             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1089             PVRSRV_HAP_SINGLE_PROCESS;
1090         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].pszName = "PixelShaderUSSE";
1091         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].pszBSName =
1092             "PixelShaderUSSE BS";
1093         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].DevMemHeapType =
1094             DEVICE_MEMORY_HEAP_PERCONTEXT;
1095
1096         psDeviceMemoryHeap[SGX_PIXELSHADER_HEAP_ID].ui32DataPageSize =
1097             SGX_MMU_PAGE_SIZE;
1098
1099         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32HeapID =
1100             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_VERTEXSHADER_HEAP_ID);
1101         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].sDevVAddrBase.uiAddr =
1102             SGX_VERTEXSHADER_HEAP_BASE;
1103         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32HeapSize =
1104             SGX_VERTEXSHADER_HEAP_SIZE;
1105         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32Attribs =
1106             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1107             PVRSRV_HAP_SINGLE_PROCESS;
1108         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].pszName =
1109             "VertexShaderUSSE";
1110         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].pszBSName =
1111             "VertexShaderUSSE BS";
1112         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].DevMemHeapType =
1113             DEVICE_MEMORY_HEAP_PERCONTEXT;
1114
1115         psDeviceMemoryHeap[SGX_VERTEXSHADER_HEAP_ID].ui32DataPageSize =
1116             SGX_MMU_PAGE_SIZE;
1117
1118         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32HeapID =
1119             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_PDSPIXEL_CODEDATA_HEAP_ID);
1120         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].sDevVAddrBase.uiAddr =
1121             SGX_PDSPIXEL_CODEDATA_HEAP_BASE;
1122         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32HeapSize =
1123             SGX_PDSPIXEL_CODEDATA_HEAP_SIZE;
1124         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32Attribs =
1125             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1126             PVRSRV_HAP_SINGLE_PROCESS;
1127         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].pszName =
1128             "PDSPixelCodeData";
1129         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].pszBSName =
1130             "PDSPixelCodeData BS";
1131         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].DevMemHeapType =
1132             DEVICE_MEMORY_HEAP_PERCONTEXT;
1133
1134         psDeviceMemoryHeap[SGX_PDSPIXEL_CODEDATA_HEAP_ID].ui32DataPageSize =
1135             SGX_MMU_PAGE_SIZE;
1136
1137         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32HeapID =
1138             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_PDSVERTEX_CODEDATA_HEAP_ID);
1139         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].sDevVAddrBase.
1140             uiAddr = SGX_PDSVERTEX_CODEDATA_HEAP_BASE;
1141         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32HeapSize =
1142             SGX_PDSVERTEX_CODEDATA_HEAP_SIZE;
1143         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32Attribs =
1144             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1145             PVRSRV_HAP_SINGLE_PROCESS;
1146         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].pszName =
1147             "PDSVertexCodeData";
1148         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].pszBSName =
1149             "PDSVertexCodeData BS";
1150         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].DevMemHeapType =
1151             DEVICE_MEMORY_HEAP_PERCONTEXT;
1152
1153         psDeviceMemoryHeap[SGX_PDSVERTEX_CODEDATA_HEAP_ID].ui32DataPageSize =
1154             SGX_MMU_PAGE_SIZE;
1155
1156         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32HeapID =
1157             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_SYNCINFO_HEAP_ID);
1158         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].sDevVAddrBase.uiAddr =
1159             SGX_SYNCINFO_HEAP_BASE;
1160         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32HeapSize =
1161             SGX_SYNCINFO_HEAP_SIZE;
1162
1163         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32Attribs =
1164             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1165             PVRSRV_HAP_MULTI_PROCESS;
1166         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].pszName = "CacheCoherent";
1167         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].pszBSName = "CacheCoherent BS";
1168
1169         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].DevMemHeapType =
1170             DEVICE_MEMORY_HEAP_SHARED_EXPORTED;
1171
1172         psDeviceMemoryHeap[SGX_SYNCINFO_HEAP_ID].ui32DataPageSize =
1173             SGX_MMU_PAGE_SIZE;
1174
1175         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32HeapID =
1176             HEAP_ID(PVRSRV_DEVICE_TYPE_SGX, SGX_3DPARAMETERS_HEAP_ID);
1177         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].sDevVAddrBase.uiAddr =
1178             SGX_3DPARAMETERS_HEAP_BASE;
1179         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32HeapSize =
1180             SGX_3DPARAMETERS_HEAP_SIZE;
1181         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].pszName = "3DParameters";
1182         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].pszBSName =
1183             "3DParameters BS";
1184         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32Attribs =
1185             PVRSRV_HAP_WRITECOMBINE | PVRSRV_MEM_RAM_BACKED_ALLOCATION |
1186             PVRSRV_HAP_SINGLE_PROCESS;
1187         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].DevMemHeapType =
1188             DEVICE_MEMORY_HEAP_PERCONTEXT;
1189
1190         psDeviceMemoryHeap[SGX_3DPARAMETERS_HEAP_ID].ui32DataPageSize =
1191             SGX_MMU_PAGE_SIZE;
1192
1193         return PVRSRV_OK;
1194 }
1195
1196 enum PVRSRV_ERROR SGXGetClientInfoKM(void *hDevCookie,
1197                                          struct SGX_CLIENT_INFO *psClientInfo)
1198 {
1199         struct PVRSRV_SGXDEV_INFO *psDevInfo =
1200             (struct PVRSRV_SGXDEV_INFO *)
1201                         ((struct PVRSRV_DEVICE_NODE *)hDevCookie)->pvDevice;
1202
1203         psDevInfo->ui32ClientRefCount++;
1204 #ifdef PDUMP
1205         if (psDevInfo->ui32ClientRefCount == 1)
1206                 psDevInfo->psKernelCCBInfo->ui32CCBDumpWOff = 0;
1207 #endif
1208         psClientInfo->ui32ProcessID = OSGetCurrentProcessIDKM();
1209
1210         OSMemCopy(&psClientInfo->asDevData, &psDevInfo->asSGXDevData,
1211                   sizeof(psClientInfo->asDevData));
1212
1213         return PVRSRV_OK;
1214 }
1215
1216 enum PVRSRV_ERROR SGXDevInitCompatCheck(struct PVRSRV_DEVICE_NODE *psDeviceNode)
1217 {
1218         struct PVRSRV_SGXDEV_INFO *psDevInfo;
1219         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo;
1220         enum PVRSRV_ERROR eError;
1221 #if !defined(NO_HARDWARE)
1222         u32 opts;
1223         u32 opt_mismatch;
1224         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1225 #endif
1226
1227         if (psDeviceNode->sDevId.eDeviceType != PVRSRV_DEVICE_TYPE_SGX) {
1228                 PVR_DPF(PVR_DBG_ERROR,
1229                          "SGXDevInitCompatCheck: Device not of type SGX");
1230                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1231                 goto exit;
1232         }
1233         psDevInfo = psDeviceNode->pvDevice;
1234         psMemInfo = psDevInfo->psKernelSGXMiscMemInfo;
1235
1236 #if !defined(NO_HARDWARE)
1237
1238         eError = SGXGetBuildInfoKM(psDevInfo, psDeviceNode);
1239         if (eError != PVRSRV_OK) {
1240                 pr_err("pvr: unable to validate device DDK version\n");
1241                 goto exit;
1242         }
1243         psSGXFeatures =
1244             &((struct PVRSRV_SGX_MISCINFO_INFO *)(psMemInfo->pvLinAddrKM))->
1245                                                             sSGXFeatures;
1246         if ((psSGXFeatures->ui32DDKVersion !=
1247              ((PVRVERSION_MAJ << 16) | (PVRVERSION_MIN << 8) |
1248               PVRVERSION_BRANCH)) ||
1249              (psSGXFeatures->ui32DDKBuild != PVRVERSION_BUILD)) {
1250                 pr_err("pvr: incompatible driver DDK revision (%d)"
1251                         "/device DDK revision (%d).\n",
1252                          PVRVERSION_BUILD, psSGXFeatures->ui32DDKBuild);
1253                 eError = PVRSRV_ERROR_DDK_VERSION_MISMATCH;
1254                 goto exit;
1255         } else {
1256                 PVR_DPF(PVR_DBG_WARNING, "(Success) SGXInit: "
1257                                 "driver DDK (%ld) and device DDK (%ld) match",
1258                          PVRVERSION_BUILD, psSGXFeatures->ui32DDKBuild);
1259         }
1260
1261         opts = psSGXFeatures->ui32BuildOptions;
1262         opt_mismatch = opts ^ SGX_BUILD_OPTIONS;
1263         /* we support the ABIs both with and without EDM tracing option */
1264         opt_mismatch &= ~PVRSRV_USSE_EDM_STATUS_DEBUG_SET_OFFSET;
1265         if (opt_mismatch) {
1266                 if (SGX_BUILD_OPTIONS & opt_mismatch)
1267                         pr_err("pvr: mismatch in driver and microkernel build "
1268                                 "options; extra options present in driver: "
1269                                 "(0x%x)", SGX_BUILD_OPTIONS & opt_mismatch);
1270
1271                 if (opts & opt_mismatch)
1272                         pr_err("pvr: Mismatch in driver and microkernel build "
1273                                 "options; extra options present in "
1274                                 "microkernel: (0x%x)", opts & opt_mismatch);
1275                 eError = PVRSRV_ERROR_BUILD_MISMATCH;
1276                 goto exit;
1277         } else {
1278                 PVR_DPF(PVR_DBG_WARNING, "(Success) SGXInit: "
1279                                 "Driver and microkernel build options match.");
1280         }
1281
1282 #endif
1283         eError = PVRSRV_OK;
1284 exit:
1285         return eError;
1286 }
1287
1288 static
1289 enum PVRSRV_ERROR SGXGetBuildInfoKM(struct PVRSRV_SGXDEV_INFO *psDevInfo,
1290                                     struct PVRSRV_DEVICE_NODE *psDeviceNode)
1291 {
1292         enum PVRSRV_ERROR eError;
1293         struct SGXMKIF_COMMAND sCommandData;
1294         struct PVRSRV_SGX_MISCINFO_INFO *psSGXMiscInfoInt;
1295         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1296
1297         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo =
1298             psDevInfo->psKernelSGXMiscMemInfo;
1299
1300         if (!psMemInfo->pvLinAddrKM) {
1301                 PVR_DPF(PVR_DBG_ERROR, "SGXGetMiscInfoKM: Invalid address.");
1302                 return PVRSRV_ERROR_INVALID_PARAMS;
1303         }
1304         psSGXMiscInfoInt = psMemInfo->pvLinAddrKM;
1305         psSGXMiscInfoInt->ui32MiscInfoFlags &= ~PVRSRV_USSE_MISCINFO_READY;
1306         psSGXFeatures = &psSGXMiscInfoInt->sSGXFeatures;
1307
1308         OSMemSet(psMemInfo->pvLinAddrKM, 0,
1309                  sizeof(struct PVRSRV_SGX_MISCINFO_INFO));
1310
1311         sCommandData.ui32Data[1] = psMemInfo->sDevVAddr.uiAddr;
1312
1313         OSMemSet(psSGXFeatures, 0, sizeof(*psSGXFeatures));
1314
1315         mb();
1316
1317         eError = SGXScheduleCCBCommandKM(psDeviceNode,
1318                                          SGXMKIF_COMMAND_REQUEST_SGXMISCINFO,
1319                                          &sCommandData, KERNEL_ID, 0);
1320
1321         if (eError != PVRSRV_OK) {
1322                 PVR_DPF(PVR_DBG_ERROR,
1323                          "SGXGetMiscInfoKM: SGXScheduleCCBCommandKM failed.");
1324                 return eError;
1325         }
1326
1327 #if !defined(NO_HARDWARE)
1328         {
1329                 IMG_BOOL bTimeout = IMG_TRUE;
1330
1331                 LOOP_UNTIL_TIMEOUT(MAX_HW_TIME_US) {
1332                         if ((psSGXMiscInfoInt->
1333                              ui32MiscInfoFlags & PVRSRV_USSE_MISCINFO_READY) !=
1334                             0) {
1335                                 bTimeout = IMG_FALSE;
1336                                 break;
1337                         }
1338                 }
1339                 END_LOOP_UNTIL_TIMEOUT();
1340
1341                 if (bTimeout)
1342                         return PVRSRV_ERROR_TIMEOUT;
1343         }
1344 #endif
1345
1346         return PVRSRV_OK;
1347 }
1348
1349 enum PVRSRV_ERROR SGXGetMiscInfoKM(struct PVRSRV_SGXDEV_INFO *psDevInfo,
1350                                        struct SGX_MISC_INFO *psMiscInfo,
1351                                        struct PVRSRV_DEVICE_NODE *psDeviceNode)
1352 {
1353         switch (psMiscInfo->eRequest) {
1354         case SGX_MISC_INFO_REQUEST_CLOCKSPEED:
1355                 {
1356                         psMiscInfo->uData.ui32SGXClockSpeed =
1357                             psDevInfo->ui32CoreClockSpeed;
1358                         return PVRSRV_OK;
1359                 }
1360
1361         case SGX_MISC_INFO_REQUEST_SGXREV:
1362                 {
1363                         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1364                         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo =
1365                             psDevInfo->psKernelSGXMiscMemInfo;
1366
1367                         SGXGetBuildInfoKM(psDevInfo, psDeviceNode);
1368                         psSGXFeatures =
1369                             &((struct PVRSRV_SGX_MISCINFO_INFO *)(psMemInfo->
1370                                                   pvLinAddrKM))->sSGXFeatures;
1371
1372                         psMiscInfo->uData.sSGXFeatures = *psSGXFeatures;
1373
1374                         PVR_DPF(PVR_DBG_MESSAGE, "SGXGetMiscInfoKM: "
1375                                         "Core 0x%lx, sw ID 0x%lx, "
1376                                         "sw Rev 0x%lx\n",
1377                                  psSGXFeatures->ui32CoreRev,
1378                                  psSGXFeatures->ui32CoreIdSW,
1379                                  psSGXFeatures->ui32CoreRevSW);
1380                         PVR_DPF(PVR_DBG_MESSAGE, "SGXGetMiscInfoKM: "
1381                                         "DDK version 0x%lx, DDK build 0x%lx\n",
1382                                  psSGXFeatures->ui32DDKVersion,
1383                                  psSGXFeatures->ui32DDKBuild);
1384
1385                         return PVRSRV_OK;
1386                 }
1387
1388         case SGX_MISC_INFO_REQUEST_DRIVER_SGXREV:
1389                 {
1390                         struct PVRSRV_KERNEL_MEM_INFO *psMemInfo =
1391                             psDevInfo->psKernelSGXMiscMemInfo;
1392                         struct PVRSRV_SGX_MISCINFO_FEATURES *psSGXFeatures;
1393
1394                         psSGXFeatures = &((struct PVRSRV_SGX_MISCINFO_INFO *)(
1395                                         psMemInfo->pvLinAddrKM))->sSGXFeatures;
1396
1397                         OSMemSet(psMemInfo->pvLinAddrKM, 0,
1398                                  sizeof(struct PVRSRV_SGX_MISCINFO_INFO));
1399
1400                         psSGXFeatures->ui32DDKVersion =
1401                             (PVRVERSION_MAJ << 16) |
1402                             (PVRVERSION_MIN << 8) | PVRVERSION_BRANCH;
1403                         psSGXFeatures->ui32DDKBuild = PVRVERSION_BUILD;
1404
1405                         psMiscInfo->uData.sSGXFeatures = *psSGXFeatures;
1406                         return PVRSRV_OK;
1407                 }
1408
1409         case SGX_MISC_INFO_REQUEST_SET_HWPERF_STATUS:
1410                 {
1411                         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1412                             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1413                         unsigned ui32MatchingFlags;
1414
1415                         if ((psMiscInfo->uData.ui32NewHWPerfStatus &
1416                              ~(PVRSRV_SGX_HWPERF_GRAPHICS_ON |
1417                                PVRSRV_SGX_HWPERF_MK_EXECUTION_ON)) != 0) {
1418                                 return PVRSRV_ERROR_INVALID_PARAMS;
1419                         }
1420
1421                         pvr_dev_lock();
1422                         ui32MatchingFlags = readl(&psDevInfo->
1423                                                  psSGXHostCtl->ui32HWPerfFlags);
1424                         ui32MatchingFlags &=
1425                                 psMiscInfo->uData.ui32NewHWPerfStatus;
1426                         if ((ui32MatchingFlags & PVRSRV_SGX_HWPERF_GRAPHICS_ON)
1427                             == 0UL) {
1428                                 psHWPerfCB->ui32OrdinalGRAPHICS = 0xffffffff;
1429                         }
1430                         if ((ui32MatchingFlags &
1431                              PVRSRV_SGX_HWPERF_MK_EXECUTION_ON) == 0UL) {
1432                                 psHWPerfCB->ui32OrdinalMK_EXECUTION =
1433                                     0xffffffffUL;
1434                         }
1435
1436
1437                         writel(psMiscInfo->uData.ui32NewHWPerfStatus,
1438                                 &psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1439                         pvr_dev_unlock();
1440 #if defined(PDUMP)
1441                         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS,
1442                                               "SGX ukernel HWPerf status %u\n",
1443                                               readl(&psDevInfo->psSGXHostCtl->
1444                                                               ui32HWPerfFlags));
1445                         PDUMPMEM(NULL, psDevInfo->psKernelSGXHostCtlMemInfo,
1446                                  offsetof(struct SGXMKIF_HOST_CTL,
1447                                           ui32HWPerfFlags),
1448                                  sizeof(psDevInfo->psSGXHostCtl->
1449                                         ui32HWPerfFlags),
1450                                  PDUMP_FLAGS_CONTINUOUS,
1451                                  MAKEUNIQUETAG(psDevInfo->
1452                                                psKernelSGXHostCtlMemInfo));
1453 #endif
1454
1455                         return PVRSRV_OK;
1456                 }
1457         case SGX_MISC_INFO_REQUEST_HWPERF_CB_ON:
1458                 {
1459
1460                         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1461                             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1462                         u32 l;
1463
1464                         psHWPerfCB->ui32OrdinalGRAPHICS = 0xffffffffUL;
1465
1466                         pvr_dev_lock();;
1467                         l = readl(&psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1468                         l |= PVRSRV_SGX_HWPERF_GRAPHICS_ON;
1469                         writel(l, &psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1470                         pvr_dev_unlock();
1471
1472                         return PVRSRV_OK;
1473                 }
1474         case SGX_MISC_INFO_REQUEST_HWPERF_CB_OFF:
1475                 {
1476                         pvr_dev_lock();
1477                         writel(0, &psDevInfo->psSGXHostCtl->ui32HWPerfFlags);
1478                         pvr_dev_unlock();
1479
1480                         return PVRSRV_OK;
1481                 }
1482         case SGX_MISC_INFO_REQUEST_HWPERF_RETRIEVE_CB:
1483                 {
1484                         struct SGX_MISC_INFO_HWPERF_RETRIEVE_CB *psRetrieve =
1485                             &psMiscInfo->uData.sRetrieveCB;
1486                         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1487                             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1488                         unsigned i;
1489
1490                         for (i = 0;
1491                              psHWPerfCB->ui32Woff != psHWPerfCB->ui32Roff
1492                              && i < psRetrieve->ui32ArraySize; i++) {
1493                                 struct SGXMKIF_HWPERF_CB_ENTRY *psData =
1494                                     &psHWPerfCB->psHWPerfCBData[psHWPerfCB->
1495                                                                 ui32Roff];
1496
1497                                 psRetrieve->psHWPerfData[i].ui32FrameNo =
1498                                     psData->ui32FrameNo;
1499                                 psRetrieve->psHWPerfData[i].ui32Type =
1500                                     (psData->ui32Type &
1501                                      PVRSRV_SGX_HWPERF_TYPE_OP_MASK);
1502                                 psRetrieve->psHWPerfData[i].ui32StartTime =
1503                                     psData->ui32Time;
1504                                 psRetrieve->psHWPerfData[i].ui32StartTimeWraps =
1505                                     psData->ui32TimeWraps;
1506                                 psRetrieve->psHWPerfData[i].ui32EndTime =
1507                                     psData->ui32Time;
1508                                 psRetrieve->psHWPerfData[i].ui32EndTimeWraps =
1509                                     psData->ui32TimeWraps;
1510                                 psRetrieve->psHWPerfData[i].ui32ClockSpeed =
1511                                     psDevInfo->ui32CoreClockSpeed;
1512                                 psRetrieve->psHWPerfData[i].ui32TimeMax =
1513                                     psDevInfo->ui32uKernelTimerClock;
1514                                 psHWPerfCB->ui32Roff =
1515                                     (psHWPerfCB->ui32Roff + 1) &
1516                                     (SGXMKIF_HWPERF_CB_SIZE - 1);
1517                         }
1518                         psRetrieve->ui32DataCount = i;
1519                         psRetrieve->ui32Time = OSClockus();
1520                         return PVRSRV_OK;
1521                 }
1522         default:
1523                 {
1524                         return PVRSRV_ERROR_INVALID_PARAMS;
1525                 }
1526         }
1527 }
1528
1529
1530
1531 static bool sgxps_active;
1532 static unsigned long sgxps_timeout;
1533
1534 IMG_BOOL isSGXPerfServerActive(void)
1535 {
1536         if (!sgxps_active)
1537                 return 0;
1538
1539         if (time_before_eq((unsigned long)OSClockus(), sgxps_timeout))
1540                 return 1;
1541
1542         sgxps_active = false;
1543         PVR_DPF(DBGPRIV_WARNING, "pvr: perf server inactive\n");
1544
1545         return 0;
1546 }
1547
1548
1549 void SGXPerfServerMonitor(u32 u32TimeStamp)
1550 {
1551         if (!sgxps_active) {
1552                 PVR_DPF(DBGPRIV_WARNING, "pvr: perf server active\n");
1553                 sgxps_active = true;
1554         }
1555
1556         /* turn off after 1 second of inactivity */
1557         sgxps_timeout = u32TimeStamp + 1000000;
1558 }
1559
1560
1561 enum PVRSRV_ERROR SGXReadDiffCountersKM(void *hDevHandle, u32 ui32Reg,
1562                                    u32 *pui32Old, IMG_BOOL bNew, u32 ui32New,
1563                                    u32 ui32NewReset, u32 ui32CountersReg,
1564                                    u32 *pui32Time, IMG_BOOL *pbActive,
1565                                    struct PVRSRV_SGXDEV_DIFF_INFO *psDiffs)
1566 {
1567         struct SYS_DATA *psSysData;
1568         struct PVRSRV_POWER_DEV *psPowerDevice;
1569         IMG_BOOL bPowered = IMG_FALSE;
1570         struct PVRSRV_DEVICE_NODE *psDeviceNode = hDevHandle;
1571         struct PVRSRV_SGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
1572
1573         if (bNew)
1574                 psDevInfo->ui32HWGroupRequested = ui32New;
1575         psDevInfo->ui32HWReset |= ui32NewReset;
1576
1577         if (SysAcquireData(&psSysData) != PVRSRV_OK)
1578                 return PVRSRV_ERROR_GENERIC;
1579
1580         psPowerDevice = psSysData->psPowerDeviceList;
1581         while (psPowerDevice) {
1582                 if (psPowerDevice->ui32DeviceIndex ==
1583                     psDeviceNode->sDevId.ui32DeviceIndex) {
1584                         bPowered =
1585                             (IMG_BOOL)(psPowerDevice->eCurrentPowerState ==
1586                                         PVRSRV_POWER_STATE_D0);
1587                         break;
1588                 }
1589
1590                 psPowerDevice = psPowerDevice->psNext;
1591         }
1592
1593         *pbActive = bPowered;
1594
1595         pvr_dev_lock();
1596
1597         {
1598                 struct PVRSRV_SGXDEV_DIFF_INFO sNew,
1599                                                *psPrev = &psDevInfo->sDiffInfo;
1600                 u32 i;
1601
1602                 sNew.ui32Time[0] = OSClockus();
1603                 *pui32Time = sNew.ui32Time[0];
1604                 if (sNew.ui32Time[0] != psPrev->ui32Time[0] && bPowered) {
1605
1606                         SGXPerfServerMonitor(*pui32Time);
1607
1608                         *pui32Old =
1609                             OSReadHWReg(psDevInfo->pvRegsBaseKM, ui32Reg);
1610
1611                         for (i = 0; i < PVRSRV_SGX_DIFF_NUM_COUNTERS; ++i) {
1612                                 sNew.aui32Counters[i] =
1613                                     OSReadHWReg(psDevInfo->pvRegsBaseKM,
1614                                                 ui32CountersReg + (i * 4));
1615                         }
1616
1617                         if (psDevInfo->ui32HWGroupRequested != *pui32Old) {
1618                                 if (psDevInfo->ui32HWReset != 0) {
1619                                         OSWriteHWReg(psDevInfo->pvRegsBaseKM,
1620                                                      ui32Reg,
1621                                                      psDevInfo->
1622                                                      ui32HWGroupRequested |
1623                                                      psDevInfo->ui32HWReset);
1624                                         psDevInfo->ui32HWReset = 0;
1625                                 }
1626                                 OSWriteHWReg(psDevInfo->pvRegsBaseKM, ui32Reg,
1627                                              psDevInfo->ui32HWGroupRequested);
1628                         }
1629
1630                         sNew.ui32Marker[0] = psDevInfo->ui32KickTACounter;
1631                         sNew.ui32Marker[1] = psDevInfo->ui32KickTARenderCounter;
1632
1633                         sNew.ui32Time[1] = readl(
1634                                 &psDevInfo->psSGXHostCtl->ui32TimeWraps);
1635
1636                         for (i = 0; i < PVRSRV_SGX_DIFF_NUM_COUNTERS; ++i) {
1637                                 psDiffs->aui32Counters[i] =
1638                                     sNew.aui32Counters[i] -
1639                                     psPrev->aui32Counters[i];
1640                         }
1641
1642                         psDiffs->ui32Marker[0] =
1643                             sNew.ui32Marker[0] - psPrev->ui32Marker[0];
1644                         psDiffs->ui32Marker[1] =
1645                             sNew.ui32Marker[1] - psPrev->ui32Marker[1];
1646
1647                         psDiffs->ui32Time[0] =
1648                             sNew.ui32Time[0] - psPrev->ui32Time[0];
1649                         psDiffs->ui32Time[1] =
1650                             sNew.ui32Time[1] - psPrev->ui32Time[1];
1651
1652                         *psPrev = sNew;
1653                 } else {
1654                         for (i = 0; i < PVRSRV_SGX_DIFF_NUM_COUNTERS; ++i)
1655                                 psDiffs->aui32Counters[i] = 0;
1656
1657                         psDiffs->ui32Marker[0] = 0;
1658                         psDiffs->ui32Marker[1] = 0;
1659
1660                         psDiffs->ui32Time[0] = 0;
1661                         psDiffs->ui32Time[1] = 0;
1662                 }
1663         }
1664
1665         SGXTestActivePowerEvent(psDeviceNode);
1666
1667         pvr_dev_unlock();
1668
1669         return PVRSRV_OK;
1670 }
1671
1672 enum PVRSRV_ERROR SGXReadHWPerfCBKM(void *hDevHandle, u32 ui32ArraySize,
1673                         struct PVRSRV_SGX_HWPERF_CB_ENTRY *psClientHWPerfEntry,
1674                         u32 *pui32DataCount, u32 *pui32ClockSpeed,
1675                         u32 *pui32HostTimeStamp)
1676 {
1677         enum PVRSRV_ERROR eError = PVRSRV_OK;
1678         struct PVRSRV_DEVICE_NODE *psDeviceNode = hDevHandle;
1679         struct PVRSRV_SGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
1680         struct SGXMKIF_HWPERF_CB *psHWPerfCB =
1681             psDevInfo->psKernelHWPerfCBMemInfo->pvLinAddrKM;
1682         unsigned i;
1683
1684         for (i = 0;
1685              psHWPerfCB->ui32Woff != psHWPerfCB->ui32Roff && i < ui32ArraySize;
1686              i++) {
1687                 struct SGXMKIF_HWPERF_CB_ENTRY *psMKPerfEntry =
1688                     &psHWPerfCB->psHWPerfCBData[psHWPerfCB->ui32Roff];
1689
1690                 psClientHWPerfEntry[i].ui32FrameNo = psMKPerfEntry->ui32FrameNo;
1691                 psClientHWPerfEntry[i].ui32Type = psMKPerfEntry->ui32Type;
1692                 psClientHWPerfEntry[i].ui32Ordinal = psMKPerfEntry->ui32Ordinal;
1693                 psClientHWPerfEntry[i].ui32Clocksx16 =
1694                     SGXConvertTimeStamp(psDevInfo, psMKPerfEntry->ui32TimeWraps,
1695                                         psMKPerfEntry->ui32Time);
1696                 OSMemCopy(&psClientHWPerfEntry[i].ui32Counters[0],
1697                           &psMKPerfEntry->ui32Counters[0],
1698                           sizeof(psMKPerfEntry->ui32Counters));
1699
1700                 psHWPerfCB->ui32Roff =
1701                     (psHWPerfCB->ui32Roff + 1) & (SGXMKIF_HWPERF_CB_SIZE - 1);
1702         }
1703
1704         *pui32DataCount = i;
1705         *pui32ClockSpeed = psDevInfo->ui32CoreClockSpeed;
1706         *pui32HostTimeStamp = OSClockus();
1707
1708         return eError;
1709 }