pvr/mm.c

   1 /**********************************************************************
   2  *
   3  * Copyright(c) 2008 Imagination Technologies Ltd. All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful but, except
  10  * as otherwise stated in writing, without any warranty; without even the
  11  * implied warranty of merchantability or fitness for a particular purpose.
  12  * See the GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * Imagination Technologies Ltd. <gpl-support@imgtec.com>
  23  * Home Park Estate, Kings Langley, Herts, WD4 8LZ, UK
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/version.h>
  28 #include <linux/mm.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/io.h>
  31 #include <linux/slab.h>
  32 #include <linux/highmem.h>
  33 #include <linux/sched.h>
  34 #include <linux/dma-mapping.h>
  35
  36 #include "img_defs.h"
  37 #include "services.h"
  38 #include "servicesint.h"
  39 #include "syscommon.h"
  40 #include "mutils.h"
  41 #include "mm.h"
  42 #include "pvrmmap.h"
  43 #include "mmap.h"
  44 #include "osfunc.h"
  45 #include "pvr_debug.h"
  46 #include "proc.h"
  47
  48 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
  49 enum DEBUG_MEM_ALLOC_TYPE {
  50         DEBUG_MEM_ALLOC_TYPE_KMALLOC,
  51         DEBUG_MEM_ALLOC_TYPE_VMALLOC,
  52         DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES,
  53         DEBUG_MEM_ALLOC_TYPE_IOREMAP,
  54         DEBUG_MEM_ALLOC_TYPE_IO,
  55         DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE,
  56         DEBUG_MEM_ALLOC_TYPE_COUNT
  57 };
  58
  59 struct DEBUG_MEM_ALLOC_REC {
  60         enum DEBUG_MEM_ALLOC_TYPE eAllocType;
  61         void *pvKey;
  62         void *pvCpuVAddr;
  63         u32 ulCpuPAddr;
  64         void *pvPrivateData;
  65         u32 ui32Bytes;
  66         pid_t pid;
  67         char *pszFileName;
  68         u32 ui32Line;
  69
  70         struct DEBUG_MEM_ALLOC_REC *psNext;
  71 };
  72
  73 static struct DEBUG_MEM_ALLOC_REC *g_MemoryRecords;
  74
  75 static u32 g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_COUNT];
  76 static u32 g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_COUNT];
  77
  78 static u32 g_SysRAMWaterMark;
  79 static u32 g_SysRAMHighWaterMark;
  80
  81 static u32 g_IOMemWaterMark;
  82 static u32 g_IOMemHighWaterMark;
  83
  84 static void DebugMemAllocRecordAdd(enum DEBUG_MEM_ALLOC_TYPE eAllocType,
  85                                        void *pvKey, void *pvCpuVAddr,
  86                                        u32 ulCpuPAddr, void *pvPrivateData,
  87                                        u32 ui32Bytes, char *pszFileName,
  88                                        u32 ui32Line);
  89
  90 static void DebugMemAllocRecordRemove(enum DEBUG_MEM_ALLOC_TYPE eAllocType,
  91                                           void *pvKey, char *pszFileName,
  92                                           u32 ui32Line);
  93
  94 static char *DebugMemAllocRecordTypeToString(
  95                                         enum DEBUG_MEM_ALLOC_TYPE eAllocType);
  96
  97 static off_t printMemoryRecords(char *buffer, size_t size, off_t off);
  98 #endif
  99
 100 #if defined(DEBUG_LINUX_MEM_AREAS)
 101 struct DEBUG_LINUX_MEM_AREA_REC {
 102         struct LinuxMemArea *psLinuxMemArea;
 103         u32 ui32Flags;
 104         pid_t pid;
 105
 106         struct DEBUG_LINUX_MEM_AREA_REC *psNext;
 107 };
 108
 109 #if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 110 static struct mutex g_sDebugMutex;
 111 #endif
 112
 113 static struct DEBUG_LINUX_MEM_AREA_REC *g_LinuxMemAreaRecords;
 114 static u32 g_LinuxMemAreaCount;
 115 static u32 g_LinuxMemAreaWaterMark;
 116 static u32 g_LinuxMemAreaHighWaterMark;
 117
 118 static off_t printLinuxMemAreaRecords(char *buffer, size_t size, off_t off);
 119 #endif
 120
 121 static struct kmem_cache *psLinuxMemAreaCache;
 122
 123
 124 static struct LinuxMemArea *LinuxMemAreaStructAlloc(void);
 125 static void LinuxMemAreaStructFree(struct LinuxMemArea *psLinuxMemArea);
 126 #if defined(DEBUG_LINUX_MEM_AREAS)
 127 static void DebugLinuxMemAreaRecordAdd(struct LinuxMemArea *psLinuxMemArea,
 128                                            u32 ui32Flags);
 129 static struct DEBUG_LINUX_MEM_AREA_REC *DebugLinuxMemAreaRecordFind(
 130                                         struct LinuxMemArea *psLinuxMemArea);
 131 static void DebugLinuxMemAreaRecordRemove(struct LinuxMemArea *psLinuxMemArea);
 132 #endif
 133
 134 enum PVRSRV_ERROR LinuxMMInit(void)
 135 {
 136 #if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 137         mutex_init(&g_sDebugMutex);
 138 #endif
 139
 140 #if defined(DEBUG_LINUX_MEM_AREAS)
 141         {
 142                 int iStatus;
 143                 iStatus =
 144                     CreateProcReadEntry("mem_areas", printLinuxMemAreaRecords);
 145                 if (iStatus != 0)
 146                         return PVRSRV_ERROR_OUT_OF_MEMORY;
 147         }
 148 #endif
 149
 150 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 151         {
 152                 int iStatus;
 153                 iStatus = CreateProcReadEntry("meminfo", printMemoryRecords);
 154                 if (iStatus != 0)
 155                         return PVRSRV_ERROR_OUT_OF_MEMORY;
 156         }
 157 #endif
 158         psLinuxMemAreaCache =
 159             kmem_cache_create("img-mm", sizeof(struct LinuxMemArea), 0, 0,
 160                                 NULL);
 161         if (!psLinuxMemAreaCache) {
 162                 PVR_DPF(PVR_DBG_ERROR, "%s: failed to allocate kmem_cache",
 163                          __func__);
 164                 return PVRSRV_ERROR_OUT_OF_MEMORY;
 165         }
 166
 167         return PVRSRV_OK;
 168 }
 169
 170 void LinuxMMCleanup(void)
 171 {
 172 #if defined(DEBUG_LINUX_MEM_AREAS)
 173         {
 174                 struct DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord =
 175                     g_LinuxMemAreaRecords, *psNextRecord;
 176
 177                 if (g_LinuxMemAreaCount)
 178                         PVR_DPF(PVR_DBG_ERROR, "%s: BUG!: "
 179                                 "There are %d struct LinuxMemArea "
 180                                 "allocation unfreed (%ld bytes)",
 181                                  __func__, g_LinuxMemAreaCount,
 182                                  g_LinuxMemAreaWaterMark);
 183
 184                 while (psCurrentRecord) {
 185                         struct LinuxMemArea *psLinuxMemArea;
 186
 187                         psNextRecord = psCurrentRecord->psNext;
 188                         psLinuxMemArea = psCurrentRecord->psLinuxMemArea;
 189                         PVR_DPF(PVR_DBG_ERROR, "%s: BUG!: "
 190                                 "Cleaning up Linux memory area (%p), "
 191                                 "type=%s, size=%ld bytes",
 192                                  __func__, psCurrentRecord->psLinuxMemArea,
 193                                  LinuxMemAreaTypeToString(psCurrentRecord->
 194                                                           psLinuxMemArea->
 195                                                           eAreaType),
 196                                  psCurrentRecord->psLinuxMemArea->
 197                                  ui32ByteSize);
 198
 199                         LinuxMemAreaDeepFree(psLinuxMemArea);
 200
 201                         psCurrentRecord = psNextRecord;
 202                 }
 203                 RemoveProcEntry("mem_areas");
 204         }
 205 #endif
 206
 207 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 208         {
 209                 struct DEBUG_MEM_ALLOC_REC *psCurrentRecord =
 210                     g_MemoryRecords, *psNextRecord;
 211
 212                 while (psCurrentRecord) {
 213                         psNextRecord = psCurrentRecord->psNext;
 214                         PVR_DPF(PVR_DBG_ERROR, "%s: BUG!: Cleaning up memory: "
 215                                  "type=%s CpuVAddr=%p CpuPAddr=0x%08lx, "
 216                                  "allocated @ file=%s,line=%d",
 217                                  __func__,
 218                                  DebugMemAllocRecordTypeToString
 219                                  (psCurrentRecord->eAllocType),
 220                                  psCurrentRecord->pvCpuVAddr,
 221                                  psCurrentRecord->ulCpuPAddr,
 222                                  psCurrentRecord->pszFileName,
 223                                  psCurrentRecord->ui32Line);
 224                         switch (psCurrentRecord->eAllocType) {
 225                         case DEBUG_MEM_ALLOC_TYPE_KMALLOC:
 226                                 KFreeWrapper(psCurrentRecord->pvCpuVAddr);
 227                                 break;
 228                         case DEBUG_MEM_ALLOC_TYPE_IOREMAP:
 229                                 IOUnmapWrapper((__force __iomem void *)
 230                                                 psCurrentRecord->pvCpuVAddr);
 231                                 break;
 232                         case DEBUG_MEM_ALLOC_TYPE_IO:
 233
 234                                 DebugMemAllocRecordRemove
 235                                     (DEBUG_MEM_ALLOC_TYPE_IO,
 236                                      psCurrentRecord->pvKey, __FILE__,
 237                                      __LINE__);
 238                                 break;
 239                         case DEBUG_MEM_ALLOC_TYPE_VMALLOC:
 240                                 VFreeWrapper(psCurrentRecord->pvCpuVAddr);
 241                                 break;
 242                         case DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES:
 243
 244                                 DebugMemAllocRecordRemove
 245                                     (DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES,
 246                                      psCurrentRecord->pvKey, __FILE__,
 247                                      __LINE__);
 248                                 break;
 249                         case DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE:
 250                                 KMemCacheFreeWrapper(psCurrentRecord->
 251                                                              pvPrivateData,
 252                                                      psCurrentRecord->
 253                                                              pvCpuVAddr);
 254                                 break;
 255                         default:
 256                                 PVR_ASSERT(0);
 257                         }
 258                         psCurrentRecord = psNextRecord;
 259                 }
 260                 RemoveProcEntry("meminfo");
 261         }
 262 #endif
 263
 264         if (psLinuxMemAreaCache) {
 265                 kmem_cache_destroy(psLinuxMemAreaCache);
 266                 psLinuxMemAreaCache = NULL;
 267         }
 268 }
 269
 270 void *_KMallocWrapper(u32 ui32ByteSize, char *pszFileName,
 271                           u32 ui32Line)
 272 {
 273         void *pvRet;
 274         pvRet = kmalloc(ui32ByteSize, GFP_KERNEL);
 275 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 276         if (pvRet)
 277                 DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_KMALLOC,
 278                                        pvRet, pvRet, 0, NULL, ui32ByteSize,
 279                                        pszFileName, ui32Line);
 280 #endif
 281         return pvRet;
 282 }
 283
 284 void _KFreeWrapper(void *pvCpuVAddr, char *pszFileName,
 285               u32 ui32Line)
 286 {
 287 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 288         DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_KMALLOC, pvCpuVAddr,
 289                                   pszFileName, ui32Line);
 290 #endif
 291         kfree(pvCpuVAddr);
 292 }
 293
 294 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 295 static void DebugMemAllocRecordAdd(enum DEBUG_MEM_ALLOC_TYPE eAllocType,
 296                                    void *pvKey, void *pvCpuVAddr,
 297                                    u32 ulCpuPAddr, void *pvPrivateData,
 298                                    u32 ui32Bytes, char *pszFileName,
 299                                    u32 ui32Line)
 300 {
 301         struct DEBUG_MEM_ALLOC_REC *psRecord;
 302
 303         mutex_lock(&g_sDebugMutex);
 304
 305         psRecord = kmalloc(sizeof(struct DEBUG_MEM_ALLOC_REC), GFP_KERNEL);
 306
 307         psRecord->eAllocType = eAllocType;
 308         psRecord->pvKey = pvKey;
 309         psRecord->pvCpuVAddr = pvCpuVAddr;
 310         psRecord->ulCpuPAddr = ulCpuPAddr;
 311         psRecord->pvPrivateData = pvPrivateData;
 312         psRecord->pid = current->pid;
 313         psRecord->ui32Bytes = ui32Bytes;
 314         psRecord->pszFileName = pszFileName;
 315         psRecord->ui32Line = ui32Line;
 316
 317         psRecord->psNext = g_MemoryRecords;
 318         g_MemoryRecords = psRecord;
 319
 320         g_WaterMarkData[eAllocType] += ui32Bytes;
 321         if (g_WaterMarkData[eAllocType] > g_HighWaterMarkData[eAllocType])
 322                 g_HighWaterMarkData[eAllocType] = g_WaterMarkData[eAllocType];
 323
 324         if (eAllocType == DEBUG_MEM_ALLOC_TYPE_KMALLOC ||
 325             eAllocType == DEBUG_MEM_ALLOC_TYPE_VMALLOC ||
 326             eAllocType == DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES ||
 327             eAllocType == DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE) {
 328                 g_SysRAMWaterMark += ui32Bytes;
 329                 if (g_SysRAMWaterMark > g_SysRAMHighWaterMark)
 330                         g_SysRAMHighWaterMark = g_SysRAMWaterMark;
 331         } else if (eAllocType == DEBUG_MEM_ALLOC_TYPE_IOREMAP ||
 332                    eAllocType == DEBUG_MEM_ALLOC_TYPE_IO) {
 333                 g_IOMemWaterMark += ui32Bytes;
 334                 if (g_IOMemWaterMark > g_IOMemHighWaterMark)
 335                         g_IOMemHighWaterMark = g_IOMemWaterMark;
 336         }
 337
 338         mutex_unlock(&g_sDebugMutex);
 339 }
 340
 341 static void DebugMemAllocRecordRemove(enum DEBUG_MEM_ALLOC_TYPE eAllocType,
 342                                       void *pvKey, char *pszFileName,
 343                                       u32 ui32Line)
 344 {
 345         struct DEBUG_MEM_ALLOC_REC **ppsCurrentRecord;
 346
 347         mutex_lock(&g_sDebugMutex);
 348
 349         for (ppsCurrentRecord = &g_MemoryRecords; *ppsCurrentRecord;
 350              ppsCurrentRecord = &((*ppsCurrentRecord)->psNext))
 351                 if ((*ppsCurrentRecord)->eAllocType == eAllocType &&
 352                     (*ppsCurrentRecord)->pvKey == pvKey) {
 353                         struct DEBUG_MEM_ALLOC_REC *psNextRecord;
 354
 355                         psNextRecord = (*ppsCurrentRecord)->psNext;
 356                         g_WaterMarkData[eAllocType] -=
 357                             (*ppsCurrentRecord)->ui32Bytes;
 358
 359                         if (eAllocType == DEBUG_MEM_ALLOC_TYPE_KMALLOC ||
 360                             eAllocType == DEBUG_MEM_ALLOC_TYPE_VMALLOC ||
 361                             eAllocType == DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES ||
 362                             eAllocType == DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE) {
 363                                 g_SysRAMWaterMark -=
 364                                     (*ppsCurrentRecord)->ui32Bytes;
 365                         } else {
 366                                 if (eAllocType == DEBUG_MEM_ALLOC_TYPE_IOREMAP
 367                                    || eAllocType == DEBUG_MEM_ALLOC_TYPE_IO)
 368                                         g_IOMemWaterMark -=
 369                                             (*ppsCurrentRecord)->ui32Bytes;
 370
 371                         }
 372
 373                         kfree(*ppsCurrentRecord);
 374                         *ppsCurrentRecord = psNextRecord;
 375                         goto exit_unlock;
 376                 }
 377
 378         PVR_DPF(PVR_DBG_ERROR, "%s: couldn't find an entry for type=%s "
 379                                 "with pvKey=%p (called from %s, line %d\n",
 380                  __func__, DebugMemAllocRecordTypeToString(eAllocType), pvKey,
 381                  pszFileName, ui32Line);
 382
 383 exit_unlock:
 384         mutex_unlock(&g_sDebugMutex);
 385 }
 386
 387 static char *DebugMemAllocRecordTypeToString(
 388                 enum DEBUG_MEM_ALLOC_TYPE eAllocType)
 389 {
 390         char *apszDebugMemoryRecordTypes[] = {
 391                 "KMALLOC",
 392                 "VMALLOC",
 393                 "ALLOC_PAGES",
 394                 "IOREMAP",
 395                 "IO",
 396                 "KMEM_CACHE_ALLOC"
 397         };
 398         return apszDebugMemoryRecordTypes[eAllocType];
 399 }
 400 #endif
 401
 402 void *_VMallocWrapper(u32 ui32Bytes, u32 ui32AllocFlags, char *pszFileName,
 403                       u32 ui32Line)
 404 {
 405         pgprot_t PGProtFlags;
 406         void *pvRet;
 407
 408         switch (ui32AllocFlags & PVRSRV_HAP_CACHETYPE_MASK) {
 409         case PVRSRV_HAP_CACHED:
 410                 PGProtFlags = PAGE_KERNEL;
 411                 break;
 412         case PVRSRV_HAP_WRITECOMBINE:
 413                 PGProtFlags = PGPROT_WC(PAGE_KERNEL);
 414                 break;
 415         case PVRSRV_HAP_UNCACHED:
 416                 PGProtFlags = PGPROT_UC(PAGE_KERNEL);
 417                 break;
 418         default:
 419                 PVR_DPF(PVR_DBG_ERROR,
 420                          "VMAllocWrapper: unknown mapping flags=0x%08lx",
 421                          ui32AllocFlags);
 422                 dump_stack();
 423                 return NULL;
 424         }
 425
 426         pvRet = __vmalloc(ui32Bytes, GFP_KERNEL | __GFP_HIGHMEM, PGProtFlags);
 427
 428 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 429         if (pvRet)
 430                 DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_VMALLOC,
 431                                        pvRet, pvRet, 0, NULL,
 432                                        PAGE_ALIGN(ui32Bytes),
 433                                        pszFileName, ui32Line);
 434 #endif
 435
 436         return pvRet;
 437 }
 438
 439 void _VFreeWrapper(void *pvCpuVAddr, char *pszFileName, u32 ui32Line)
 440 {
 441 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 442         DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_VMALLOC, pvCpuVAddr,
 443                                   pszFileName, ui32Line);
 444 #endif
 445         vfree(pvCpuVAddr);
 446 }
 447
 448 struct LinuxMemArea *NewVMallocLinuxMemArea(u32 ui32Bytes, u32 ui32AreaFlags)
 449 {
 450         struct LinuxMemArea *psLinuxMemArea;
 451         void *pvCpuVAddr;
 452
 453         psLinuxMemArea = LinuxMemAreaStructAlloc();
 454         if (!psLinuxMemArea)
 455                 goto failed;
 456
 457         pvCpuVAddr = VMallocWrapper(ui32Bytes, ui32AreaFlags);
 458         if (!pvCpuVAddr)
 459                 goto failed;
 460
 461         psLinuxMemArea->eAreaType = LINUX_MEM_AREA_VMALLOC;
 462         psLinuxMemArea->uData.sVmalloc.pvVmallocAddress = pvCpuVAddr;
 463         psLinuxMemArea->ui32ByteSize = ui32Bytes;
 464         psLinuxMemArea->ui32AreaFlags = ui32AreaFlags;
 465         psLinuxMemArea->bMMapRegistered = IMG_FALSE;
 466         INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList);
 467
 468 #if defined(DEBUG_LINUX_MEM_AREAS)
 469         DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags);
 470 #endif
 471
 472         return psLinuxMemArea;
 473
 474 failed:
 475         PVR_DPF(PVR_DBG_ERROR, "%s: failed!", __func__);
 476         if (psLinuxMemArea)
 477                 LinuxMemAreaStructFree(psLinuxMemArea);
 478         return NULL;
 479 }
 480
 481 void FreeVMallocLinuxMemArea(struct LinuxMemArea *psLinuxMemArea)
 482 {
 483         PVR_ASSERT(psLinuxMemArea);
 484         PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_VMALLOC);
 485         PVR_ASSERT(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress);
 486
 487 #if defined(DEBUG_LINUX_MEM_AREAS)
 488         DebugLinuxMemAreaRecordRemove(psLinuxMemArea);
 489 #endif
 490
 491
 492         PVR_DPF(PVR_DBG_MESSAGE, "%s: pvCpuVAddr: %p",
 493                  __func__,
 494                  psLinuxMemArea->uData.sVmalloc.pvVmallocAddress);
 495         VFreeWrapper(psLinuxMemArea->uData.sVmalloc.pvVmallocAddress);
 496
 497         LinuxMemAreaStructFree(psLinuxMemArea);
 498 }
 499
 500 void __iomem *_IORemapWrapper(struct IMG_CPU_PHYADDR BasePAddr,
 501                           u32 ui32Bytes, u32 ui32MappingFlags,
 502                           char *pszFileName, u32 ui32Line)
 503 {
 504         void __iomem *pvIORemapCookie = NULL;
 505
 506         switch (ui32MappingFlags & PVRSRV_HAP_CACHETYPE_MASK) {
 507         case PVRSRV_HAP_CACHED:
 508                 pvIORemapCookie = IOREMAP(BasePAddr.uiAddr, ui32Bytes);
 509                 break;
 510         case PVRSRV_HAP_WRITECOMBINE:
 511                 pvIORemapCookie = IOREMAP_WC(BasePAddr.uiAddr, ui32Bytes);
 512                 break;
 513         case PVRSRV_HAP_UNCACHED:
 514                 pvIORemapCookie = IOREMAP_UC(BasePAddr.uiAddr, ui32Bytes);
 515                 break;
 516         default:
 517                 PVR_DPF(PVR_DBG_ERROR,
 518                          "IORemapWrapper: unknown mapping flags");
 519                 return NULL;
 520         }
 521
 522 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 523         if (pvIORemapCookie)
 524                 DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_IOREMAP,
 525                                        (__force void *)pvIORemapCookie,
 526                                        (__force void *)pvIORemapCookie,
 527                                        BasePAddr.uiAddr,
 528                                        NULL, ui32Bytes, pszFileName, ui32Line);
 529 #endif
 530
 531         return pvIORemapCookie;
 532 }
 533
 534 void _IOUnmapWrapper(void __iomem *pvIORemapCookie, char *pszFileName,
 535                 u32 ui32Line)
 536 {
 537 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 538         DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_IOREMAP,
 539                                   (__force void *)pvIORemapCookie,
 540                                   pszFileName, ui32Line);
 541 #endif
 542         iounmap(pvIORemapCookie);
 543 }
 544
 545 struct LinuxMemArea *NewIORemapLinuxMemArea(struct IMG_CPU_PHYADDR BasePAddr,
 546                                             u32 ui32Bytes, u32 ui32AreaFlags)
 547 {
 548         struct LinuxMemArea *psLinuxMemArea;
 549         void __iomem *pvIORemapCookie;
 550
 551         psLinuxMemArea = LinuxMemAreaStructAlloc();
 552         if (!psLinuxMemArea)
 553                 return NULL;
 554
 555         pvIORemapCookie = IORemapWrapper(BasePAddr, ui32Bytes, ui32AreaFlags);
 556         if (!pvIORemapCookie) {
 557                 LinuxMemAreaStructFree(psLinuxMemArea);
 558                 return NULL;
 559         }
 560
 561         psLinuxMemArea->eAreaType = LINUX_MEM_AREA_IOREMAP;
 562         psLinuxMemArea->uData.sIORemap.pvIORemapCookie = pvIORemapCookie;
 563         psLinuxMemArea->uData.sIORemap.CPUPhysAddr = BasePAddr;
 564         psLinuxMemArea->ui32ByteSize = ui32Bytes;
 565         psLinuxMemArea->ui32AreaFlags = ui32AreaFlags;
 566         psLinuxMemArea->bMMapRegistered = IMG_FALSE;
 567         INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList);
 568
 569 #if defined(DEBUG_LINUX_MEM_AREAS)
 570         DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags);
 571 #endif
 572
 573         return psLinuxMemArea;
 574 }
 575
 576 void FreeIORemapLinuxMemArea(struct LinuxMemArea *psLinuxMemArea)
 577 {
 578         PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_IOREMAP);
 579
 580 #if defined(DEBUG_LINUX_MEM_AREAS)
 581         DebugLinuxMemAreaRecordRemove(psLinuxMemArea);
 582 #endif
 583
 584         IOUnmapWrapper(psLinuxMemArea->uData.sIORemap.pvIORemapCookie);
 585
 586         LinuxMemAreaStructFree(psLinuxMemArea);
 587 }
 588
 589 static IMG_BOOL PagesAreContiguous(struct IMG_SYS_PHYADDR *psSysPhysAddr,
 590                                    u32 ui32Bytes)
 591 {
 592         u32 ui32;
 593         u32 ui32AddrChk;
 594         u32 ui32NumPages = RANGE_TO_PAGES(ui32Bytes);
 595
 596         for (ui32 = 0, ui32AddrChk = psSysPhysAddr[0].uiAddr;
 597              ui32 < ui32NumPages; ui32++, ui32AddrChk += PAGE_SIZE)
 598                 if (psSysPhysAddr[ui32].uiAddr != ui32AddrChk)
 599                         return IMG_FALSE;
 600
 601         return IMG_TRUE;
 602 }
 603
 604 struct LinuxMemArea *NewExternalKVLinuxMemArea(struct IMG_SYS_PHYADDR
 605                                                *pBasePAddr, void *pvCPUVAddr,
 606                                                u32 ui32Bytes,
 607                                                IMG_BOOL bPhysContig,
 608                                                u32 ui32AreaFlags)
 609 {
 610         struct LinuxMemArea *psLinuxMemArea;
 611
 612         psLinuxMemArea = LinuxMemAreaStructAlloc();
 613         if (!psLinuxMemArea)
 614                 return NULL;
 615
 616         psLinuxMemArea->eAreaType = LINUX_MEM_AREA_EXTERNAL_KV;
 617         psLinuxMemArea->uData.sExternalKV.pvExternalKV = pvCPUVAddr;
 618         psLinuxMemArea->uData.sExternalKV.bPhysContig = bPhysContig ||
 619                                       PagesAreContiguous(pBasePAddr, ui32Bytes);
 620
 621         if (psLinuxMemArea->uData.sExternalKV.bPhysContig)
 622                 psLinuxMemArea->uData.sExternalKV.uPhysAddr.SysPhysAddr =
 623                     *pBasePAddr;
 624         else
 625                 psLinuxMemArea->uData.sExternalKV.uPhysAddr.pSysPhysAddr =
 626                     pBasePAddr;
 627         psLinuxMemArea->ui32ByteSize = ui32Bytes;
 628         psLinuxMemArea->ui32AreaFlags = ui32AreaFlags;
 629         psLinuxMemArea->bMMapRegistered = IMG_FALSE;
 630         INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList);
 631
 632 #if defined(DEBUG_LINUX_MEM_AREAS)
 633         DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags);
 634 #endif
 635
 636         return psLinuxMemArea;
 637 }
 638
 639 void FreeExternalKVLinuxMemArea(struct LinuxMemArea *psLinuxMemArea)
 640 {
 641         PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_EXTERNAL_KV);
 642
 643 #if defined(DEBUG_LINUX_MEM_AREAS)
 644         DebugLinuxMemAreaRecordRemove(psLinuxMemArea);
 645 #endif
 646
 647         LinuxMemAreaStructFree(psLinuxMemArea);
 648 }
 649
 650 struct LinuxMemArea *NewIOLinuxMemArea(struct IMG_CPU_PHYADDR BasePAddr,
 651                                 u32 ui32Bytes, u32 ui32AreaFlags)
 652 {
 653         struct LinuxMemArea *psLinuxMemArea = LinuxMemAreaStructAlloc();
 654         if (!psLinuxMemArea)
 655                 return NULL;
 656
 657         psLinuxMemArea->eAreaType = LINUX_MEM_AREA_IO;
 658         psLinuxMemArea->uData.sIO.CPUPhysAddr.uiAddr = BasePAddr.uiAddr;
 659         psLinuxMemArea->ui32ByteSize = ui32Bytes;
 660         psLinuxMemArea->ui32AreaFlags = ui32AreaFlags;
 661         psLinuxMemArea->bMMapRegistered = IMG_FALSE;
 662         INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList);
 663
 664 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 665         DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_IO,
 666                                (void *)BasePAddr.uiAddr, NULL,
 667                                BasePAddr.uiAddr, NULL, ui32Bytes, "unknown", 0);
 668 #endif
 669
 670 #if defined(DEBUG_LINUX_MEM_AREAS)
 671         DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags);
 672 #endif
 673
 674         return psLinuxMemArea;
 675 }
 676
 677 void FreeIOLinuxMemArea(struct LinuxMemArea *psLinuxMemArea)
 678 {
 679         PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_IO);
 680
 681 #if defined(DEBUG_LINUX_MEM_AREAS)
 682         DebugLinuxMemAreaRecordRemove(psLinuxMemArea);
 683 #endif
 684
 685 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 686         DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_IO,
 687                                   (void *)psLinuxMemArea->uData.sIO.
 688                                   CPUPhysAddr.uiAddr, __FILE__, __LINE__);
 689 #endif
 690
 691         LinuxMemAreaStructFree(psLinuxMemArea);
 692 }
 693
 694 struct LinuxMemArea *NewAllocPagesLinuxMemArea(u32 ui32Bytes,
 695                                         u32 ui32AreaFlags)
 696 {
 697         struct LinuxMemArea *psLinuxMemArea;
 698         u32 ui32PageCount;
 699         struct page **pvPageList;
 700         void *hBlockPageList;
 701         s32 i;
 702         enum PVRSRV_ERROR eError;
 703
 704         psLinuxMemArea = LinuxMemAreaStructAlloc();
 705         if (!psLinuxMemArea)
 706                 goto failed_area_alloc;
 707
 708         ui32PageCount = RANGE_TO_PAGES(ui32Bytes);
 709         eError = OSAllocMem(0, sizeof(*pvPageList) * ui32PageCount,
 710                        (void **)&pvPageList, &hBlockPageList);
 711         if (eError != PVRSRV_OK)
 712                 goto failed_page_list_alloc;
 713
 714         for (i = 0; i < ui32PageCount; i++) {
 715                 pvPageList[i] = alloc_pages(GFP_KERNEL | __GFP_HIGHMEM, 0);
 716                 if (!pvPageList[i])
 717                         goto failed_alloc_pages;
 718
 719         }
 720
 721 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 722         DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES,
 723                                pvPageList, NULL, 0, NULL, PAGE_ALIGN(ui32Bytes),
 724                                "unknown", 0);
 725 #endif
 726
 727         psLinuxMemArea->eAreaType = LINUX_MEM_AREA_ALLOC_PAGES;
 728         psLinuxMemArea->uData.sPageList.pvPageList = pvPageList;
 729         psLinuxMemArea->uData.sPageList.hBlockPageList = hBlockPageList;
 730         psLinuxMemArea->ui32ByteSize = ui32Bytes;
 731         psLinuxMemArea->ui32AreaFlags = ui32AreaFlags;
 732         psLinuxMemArea->bMMapRegistered = IMG_FALSE;
 733         INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList);
 734
 735 #if defined(DEBUG_LINUX_MEM_AREAS)
 736         DebugLinuxMemAreaRecordAdd(psLinuxMemArea, ui32AreaFlags);
 737 #endif
 738
 739         return psLinuxMemArea;
 740
 741 failed_alloc_pages:
 742         for (i--; i >= 0; i--)
 743                 __free_pages(pvPageList[i], 0);
 744         OSFreeMem(0, sizeof(*pvPageList) * ui32PageCount, pvPageList,
 745                         hBlockPageList);
 746 failed_page_list_alloc:
 747         LinuxMemAreaStructFree(psLinuxMemArea);
 748 failed_area_alloc:
 749         PVR_DPF(PVR_DBG_ERROR, "%s: failed", __func__);
 750
 751         return NULL;
 752 }
 753
 754 void FreeAllocPagesLinuxMemArea(struct LinuxMemArea *psLinuxMemArea)
 755 {
 756         u32 ui32PageCount;
 757         struct page **pvPageList;
 758         void *hBlockPageList;
 759         u32 i;
 760
 761         PVR_ASSERT(psLinuxMemArea);
 762         PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_ALLOC_PAGES);
 763
 764 #if defined(DEBUG_LINUX_MEM_AREAS)
 765         DebugLinuxMemAreaRecordRemove(psLinuxMemArea);
 766 #endif
 767
 768         ui32PageCount = RANGE_TO_PAGES(psLinuxMemArea->ui32ByteSize);
 769         pvPageList = psLinuxMemArea->uData.sPageList.pvPageList;
 770         hBlockPageList = psLinuxMemArea->uData.sPageList.hBlockPageList;
 771
 772 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 773         DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES, pvPageList,
 774                                   __FILE__, __LINE__);
 775 #endif
 776
 777         for (i = 0; i < ui32PageCount; i++)
 778                 __free_pages(pvPageList[i], 0);
 779
 780         OSFreeMem(0, sizeof(*pvPageList) * ui32PageCount, pvPageList,
 781                         hBlockPageList);
 782
 783         LinuxMemAreaStructFree(psLinuxMemArea);
 784 }
 785
 786 struct page *LinuxMemAreaOffsetToPage(struct LinuxMemArea *psLinuxMemArea,
 787                                       u32 ui32ByteOffset)
 788 {
 789         u32 ui32PageIndex;
 790         char *pui8Addr;
 791
 792         switch (psLinuxMemArea->eAreaType) {
 793         case LINUX_MEM_AREA_ALLOC_PAGES:
 794                 ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset);
 795                 return
 796                      psLinuxMemArea->uData.sPageList.pvPageList[ui32PageIndex];
 797                 break;
 798         case LINUX_MEM_AREA_VMALLOC:
 799                 pui8Addr = psLinuxMemArea->uData.sVmalloc.pvVmallocAddress;
 800                 pui8Addr += ui32ByteOffset;
 801                 return vmalloc_to_page(pui8Addr);
 802                 break;
 803         case LINUX_MEM_AREA_SUB_ALLOC:
 804                 return LinuxMemAreaOffsetToPage(psLinuxMemArea->
 805                                      uData.sSubAlloc.psParentLinuxMemArea,
 806                                      psLinuxMemArea->
 807                                                 uData.sSubAlloc.ui32ByteOffset +
 808                                                 ui32ByteOffset);
 809         default:
 810                 PVR_DPF(PVR_DBG_ERROR, "%s: Unsupported request for "
 811                          "struct page from struct LinuxMemArea with type=%s",
 812                          LinuxMemAreaTypeToString(psLinuxMemArea->eAreaType));
 813                 return NULL;
 814         }
 815 }
 816
 817 void *_KMemCacheAllocWrapper(struct kmem_cache *psCache,
 818                                  gfp_t Flags,
 819                                  char *pszFileName, u32 ui32Line)
 820 {
 821         void *pvRet;
 822
 823         pvRet = kmem_cache_alloc(psCache, Flags);
 824
 825 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 826         DebugMemAllocRecordAdd(DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE, pvRet, pvRet,
 827                                0, psCache, kmem_cache_size(psCache),
 828                                pszFileName, ui32Line);
 829 #endif
 830
 831         return pvRet;
 832 }
 833
 834 void _KMemCacheFreeWrapper(struct kmem_cache *psCache, void *pvObject,
 835                       char *pszFileName, u32 ui32Line)
 836 {
 837 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
 838         DebugMemAllocRecordRemove(DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE, pvObject,
 839                                   pszFileName, ui32Line);
 840 #endif
 841
 842         kmem_cache_free(psCache, pvObject);
 843 }
 844
 845 const char *KMemCacheNameWrapper(struct kmem_cache *psCache)
 846 {
 847
 848         return "";
 849 }
 850
 851 struct LinuxMemArea *NewSubLinuxMemArea(struct LinuxMemArea
 852                                 *psParentLinuxMemArea, u32 ui32ByteOffset,
 853                                 u32 ui32Bytes)
 854 {
 855         struct LinuxMemArea *psLinuxMemArea;
 856
 857         PVR_ASSERT((ui32ByteOffset + ui32Bytes) <=
 858                    psParentLinuxMemArea->ui32ByteSize);
 859
 860         psLinuxMemArea = LinuxMemAreaStructAlloc();
 861         if (!psLinuxMemArea)
 862                 return NULL;
 863
 864         psLinuxMemArea->eAreaType = LINUX_MEM_AREA_SUB_ALLOC;
 865         psLinuxMemArea->uData.sSubAlloc.psParentLinuxMemArea =
 866                                                         psParentLinuxMemArea;
 867         psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset = ui32ByteOffset;
 868         psLinuxMemArea->ui32ByteSize = ui32Bytes;
 869         psLinuxMemArea->ui32AreaFlags = psParentLinuxMemArea->ui32AreaFlags;
 870         psLinuxMemArea->bMMapRegistered = IMG_FALSE;
 871         INIT_LIST_HEAD(&psLinuxMemArea->sMMapOffsetStructList);
 872
 873 #if defined(DEBUG_LINUX_MEM_AREAS)
 874         {
 875                 struct DEBUG_LINUX_MEM_AREA_REC *psParentRecord;
 876                 psParentRecord =
 877                     DebugLinuxMemAreaRecordFind(psParentLinuxMemArea);
 878                 DebugLinuxMemAreaRecordAdd(psLinuxMemArea,
 879                                            psParentRecord->ui32Flags);
 880         }
 881 #endif
 882
 883         return psLinuxMemArea;
 884 }
 885
 886 static void FreeSubLinuxMemArea(struct LinuxMemArea *psLinuxMemArea)
 887 {
 888         PVR_ASSERT(psLinuxMemArea->eAreaType == LINUX_MEM_AREA_SUB_ALLOC);
 889
 890 #if defined(DEBUG_LINUX_MEM_AREAS)
 891         DebugLinuxMemAreaRecordRemove(psLinuxMemArea);
 892 #endif
 893
 894         LinuxMemAreaStructFree(psLinuxMemArea);
 895 }
 896
 897 static struct LinuxMemArea *LinuxMemAreaStructAlloc(void)
 898 {
 899         return KMemCacheAllocWrapper(psLinuxMemAreaCache, GFP_KERNEL);
 900 }
 901
 902 static void LinuxMemAreaStructFree(struct LinuxMemArea *psLinuxMemArea)
 903 {
 904         KMemCacheFreeWrapper(psLinuxMemAreaCache, psLinuxMemArea);
 905
 906 }
 907
 908 void LinuxMemAreaDeepFree(struct LinuxMemArea *psLinuxMemArea)
 909 {
 910         switch (psLinuxMemArea->eAreaType) {
 911         case LINUX_MEM_AREA_VMALLOC:
 912                 FreeVMallocLinuxMemArea(psLinuxMemArea);
 913                 break;
 914         case LINUX_MEM_AREA_ALLOC_PAGES:
 915                 FreeAllocPagesLinuxMemArea(psLinuxMemArea);
 916                 break;
 917         case LINUX_MEM_AREA_IOREMAP:
 918                 FreeIORemapLinuxMemArea(psLinuxMemArea);
 919                 break;
 920         case LINUX_MEM_AREA_EXTERNAL_KV:
 921                 FreeExternalKVLinuxMemArea(psLinuxMemArea);
 922                 break;
 923         case LINUX_MEM_AREA_IO:
 924                 FreeIOLinuxMemArea(psLinuxMemArea);
 925                 break;
 926         case LINUX_MEM_AREA_SUB_ALLOC:
 927                 FreeSubLinuxMemArea(psLinuxMemArea);
 928                 break;
 929         default:
 930                 PVR_DPF(PVR_DBG_ERROR, "%s: Unknown are type (%d)\n",
 931                          __func__, psLinuxMemArea->eAreaType);
 932         }
 933 }
 934
 935 #if defined(DEBUG_LINUX_MEM_AREAS)
 936 static void DebugLinuxMemAreaRecordAdd(struct LinuxMemArea *psLinuxMemArea,
 937                                        u32 ui32Flags)
 938 {
 939         struct DEBUG_LINUX_MEM_AREA_REC *psNewRecord;
 940         const char *pi8FlagsString;
 941
 942         mutex_lock(&g_sDebugMutex);
 943
 944         if (psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC) {
 945                 g_LinuxMemAreaWaterMark += psLinuxMemArea->ui32ByteSize;
 946                 if (g_LinuxMemAreaWaterMark > g_LinuxMemAreaHighWaterMark)
 947                         g_LinuxMemAreaHighWaterMark = g_LinuxMemAreaWaterMark;
 948         }
 949         g_LinuxMemAreaCount++;
 950
 951         psNewRecord = kmalloc(sizeof(struct DEBUG_LINUX_MEM_AREA_REC),
 952                               GFP_KERNEL);
 953         if (psNewRecord) {
 954                 psNewRecord->psLinuxMemArea = psLinuxMemArea;
 955                 psNewRecord->ui32Flags = ui32Flags;
 956                 psNewRecord->pid = current->pid;
 957                 psNewRecord->psNext = g_LinuxMemAreaRecords;
 958                 g_LinuxMemAreaRecords = psNewRecord;
 959         } else {
 960                 PVR_DPF(PVR_DBG_ERROR,
 961                          "%s: failed to allocate linux memory area record.",
 962                          __func__);
 963         }
 964
 965         pi8FlagsString = HAPFlagsToString(ui32Flags);
 966         if (strstr(pi8FlagsString, "UNKNOWN"))
 967                 PVR_DPF(PVR_DBG_ERROR, "%s: Unexpected flags "
 968                          "(0x%08lx) associated with psLinuxMemArea @ 0x%08lx",
 969                          __func__, ui32Flags, psLinuxMemArea);
 970
 971         mutex_unlock(&g_sDebugMutex);
 972 }
 973
 974 static struct DEBUG_LINUX_MEM_AREA_REC *DebugLinuxMemAreaRecordFind(
 975                                            struct LinuxMemArea *psLinuxMemArea)
 976 {
 977         struct DEBUG_LINUX_MEM_AREA_REC *psCurrentRecord;
 978
 979         mutex_lock(&g_sDebugMutex);
 980
 981         for (psCurrentRecord = g_LinuxMemAreaRecords;
 982              psCurrentRecord; psCurrentRecord = psCurrentRecord->psNext)
 983                 if (psCurrentRecord->psLinuxMemArea == psLinuxMemArea)
 984                         goto exit_unlock;
 985
 986 exit_unlock:
 987         mutex_unlock(&g_sDebugMutex);
 988
 989         return psCurrentRecord;
 990 }
 991
 992 static void DebugLinuxMemAreaRecordRemove(struct LinuxMemArea *psLinuxMemArea)
 993 {
 994         struct DEBUG_LINUX_MEM_AREA_REC **ppsCurrentRecord;
 995
 996         mutex_lock(&g_sDebugMutex);
 997
 998         if (psLinuxMemArea->eAreaType != LINUX_MEM_AREA_SUB_ALLOC)
 999                 g_LinuxMemAreaWaterMark -= psLinuxMemArea->ui32ByteSize;
1000         g_LinuxMemAreaCount--;
1001
1002         for (ppsCurrentRecord = &g_LinuxMemAreaRecords;
1003              *ppsCurrentRecord;
1004              ppsCurrentRecord = &((*ppsCurrentRecord)->psNext))
1005                 if ((*ppsCurrentRecord)->psLinuxMemArea == psLinuxMemArea) {
1006                         struct DEBUG_LINUX_MEM_AREA_REC *psNextRecord;
1007
1008                         psNextRecord = (*ppsCurrentRecord)->psNext;
1009                         kfree(*ppsCurrentRecord);
1010                         *ppsCurrentRecord = psNextRecord;
1011                         goto exit_unlock;
1012                 }
1013
1014         PVR_DPF(PVR_DBG_ERROR,
1015                  "%s: couldn't find an entry for psLinuxMemArea=%p\n", __func__,
1016                  psLinuxMemArea);
1017
1018 exit_unlock:
1019         mutex_unlock(&g_sDebugMutex);
1020 }
1021 #endif
1022
1023 void *LinuxMemAreaToCpuVAddr(struct LinuxMemArea *psLinuxMemArea)
1024 {
1025         switch (psLinuxMemArea->eAreaType) {
1026         case LINUX_MEM_AREA_VMALLOC:
1027                 return psLinuxMemArea->uData.sVmalloc.pvVmallocAddress;
1028         case LINUX_MEM_AREA_IOREMAP:
1029                 return (void __force *)
1030                         psLinuxMemArea->uData.sIORemap.pvIORemapCookie;
1031         case LINUX_MEM_AREA_EXTERNAL_KV:
1032                 return psLinuxMemArea->uData.sExternalKV.pvExternalKV;
1033         case LINUX_MEM_AREA_SUB_ALLOC:
1034                 {
1035                         char *pAddr =
1036                             LinuxMemAreaToCpuVAddr(psLinuxMemArea->uData.
1037                                                    sSubAlloc.
1038                                                          psParentLinuxMemArea);
1039                         if (!pAddr)
1040                                 return NULL;
1041                         return pAddr +
1042                                psLinuxMemArea->uData.sSubAlloc.ui32ByteOffset;
1043                 }
1044         default:
1045                 return NULL;
1046         }
1047 }
1048
1049 struct IMG_CPU_PHYADDR LinuxMemAreaToCpuPAddr(
1050                                         struct LinuxMemArea *psLinuxMemArea,
1051                                         u32 ui32ByteOffset)
1052 {
1053         struct IMG_CPU_PHYADDR CpuPAddr;
1054
1055         CpuPAddr.uiAddr = 0;
1056
1057         switch (psLinuxMemArea->eAreaType) {
1058         case LINUX_MEM_AREA_IOREMAP:
1059                 {
1060                         CpuPAddr = psLinuxMemArea->uData.sIORemap.CPUPhysAddr;
1061                         CpuPAddr.uiAddr += ui32ByteOffset;
1062                         break;
1063                 }
1064         case LINUX_MEM_AREA_EXTERNAL_KV:
1065                 {
1066                         if (psLinuxMemArea->uData.sExternalKV.bPhysContig) {
1067                                 CpuPAddr =
1068                                     SysSysPAddrToCpuPAddr(
1069                                                  psLinuxMemArea->uData.
1070                                                           sExternalKV.uPhysAddr.
1071                                                                   SysPhysAddr);
1072                                 CpuPAddr.uiAddr += ui32ByteOffset;
1073                         } else {
1074                                 u32 ui32PageIndex =
1075                                     PHYS_TO_PFN(ui32ByteOffset);
1076                                 struct IMG_SYS_PHYADDR SysPAddr =
1077                                     psLinuxMemArea->uData.sExternalKV.uPhysAddr.
1078                                                     pSysPhysAddr[ui32PageIndex];
1079
1080                                 CpuPAddr = SysSysPAddrToCpuPAddr(SysPAddr);
1081                                 CpuPAddr.uiAddr +=
1082                                     ADDR_TO_PAGE_OFFSET(ui32ByteOffset);
1083                         }
1084                         break;
1085                 }
1086         case LINUX_MEM_AREA_IO:
1087                 {
1088                         CpuPAddr = psLinuxMemArea->uData.sIO.CPUPhysAddr;
1089                         CpuPAddr.uiAddr += ui32ByteOffset;
1090                         break;
1091                 }
1092         case LINUX_MEM_AREA_VMALLOC:
1093                 {
1094                         char *pCpuVAddr;
1095                         pCpuVAddr =
1096                             (char *) psLinuxMemArea->uData.sVmalloc.
1097                             pvVmallocAddress;
1098                         pCpuVAddr += ui32ByteOffset;
1099                         CpuPAddr.uiAddr = VMallocToPhys(pCpuVAddr);
1100                         break;
1101                 }
1102         case LINUX_MEM_AREA_ALLOC_PAGES:
1103                 {
1104                         struct page *page;
1105                         u32 ui32PageIndex = PHYS_TO_PFN(ui32ByteOffset);
1106                         page =
1107                             psLinuxMemArea->uData.sPageList.
1108                             pvPageList[ui32PageIndex];
1109                         CpuPAddr.uiAddr = page_to_phys(page);
1110                         CpuPAddr.uiAddr += ADDR_TO_PAGE_OFFSET(ui32ByteOffset);
1111                         break;
1112                 }
1113         case LINUX_MEM_AREA_SUB_ALLOC:
1114                 {
1115                         CpuPAddr =
1116                             OSMemHandleToCpuPAddr(psLinuxMemArea->uData.
1117                                                   sSubAlloc.
1118                                                   psParentLinuxMemArea,
1119                                                   psLinuxMemArea->uData.
1120                                                   sSubAlloc.ui32ByteOffset +
1121                                                   ui32ByteOffset);
1122                         break;
1123                 }
1124         default:
1125                 PVR_DPF(PVR_DBG_ERROR,
1126                          "%s: Unknown struct LinuxMemArea type (%d)\n",
1127                          __func__, psLinuxMemArea->eAreaType);
1128         }
1129
1130         PVR_ASSERT(CpuPAddr.uiAddr);
1131         return CpuPAddr;
1132 }
1133
1134 static void inv_cache_vmalloc(const struct LinuxMemArea *mem_area)
1135 {
1136         struct page *pg;
1137         void *kaddr;
1138         size_t chunk;
1139         u32 pg_cnt;
1140         u32 pg_ofs;
1141         u32 vaddr, vaddr_end;
1142
1143         extern void ___dma_single_dev_to_cpu(const void *, size_t,
1144                                         enum dma_data_direction);
1145
1146         vaddr = (u32)mem_area->uData.sVmalloc.pvVmallocAddress;
1147         vaddr_end = vaddr + mem_area->ui32ByteSize;
1148         pg_cnt = (PAGE_ALIGN(vaddr_end) - (vaddr & PAGE_MASK)) / PAGE_SIZE;
1149
1150         while (pg_cnt--) {
1151                 pg = pfn_to_page(VMallocToPhys((void *)vaddr) >> PAGE_SHIFT);
1152                 kaddr = page_address(pg);
1153                 pg_ofs = vaddr & ~PAGE_MASK;
1154                 kaddr += pg_ofs;
1155                 chunk = min_t(ssize_t, vaddr_end - vaddr, PAGE_SIZE - pg_ofs);
1156                 ___dma_single_dev_to_cpu(kaddr, chunk, DMA_FROM_DEVICE);
1157                 vaddr += chunk;
1158         }
1159 }
1160
1161 static void inv_cache_page_list(const struct LinuxMemArea *mem_area)
1162 {
1163         u32 pg_cnt;
1164         struct page **pg_list;
1165
1166         extern void ___dma_single_dev_to_cpu(const void *, size_t,
1167                                         enum dma_data_direction);
1168
1169         pg_cnt = RANGE_TO_PAGES(mem_area->ui32ByteSize);
1170         pg_list = mem_area->uData.sPageList.pvPageList;
1171         while (pg_cnt--)
1172                 ___dma_single_dev_to_cpu(page_address(*pg_list++), PAGE_SIZE,
1173                                 DMA_FROM_DEVICE);
1174 }
1175
1176 void inv_cache_mem_area(const struct LinuxMemArea *mem_area)
1177 {
1178         switch (mem_area->eAreaType) {
1179         case LINUX_MEM_AREA_VMALLOC:
1180                 inv_cache_vmalloc(mem_area);
1181                 break;
1182         case LINUX_MEM_AREA_ALLOC_PAGES:
1183                 inv_cache_page_list(mem_area);
1184                 break;
1185         case LINUX_MEM_AREA_IOREMAP:
1186         case LINUX_MEM_AREA_EXTERNAL_KV:
1187         case LINUX_MEM_AREA_IO:
1188         case LINUX_MEM_AREA_SUB_ALLOC:
1189                 PVR_DPF(PVR_DBG_ERROR,
1190                         "%s: Not implemented for type (%d)\n",
1191                         __func__, mem_area->eAreaType);
1192                 BUG();
1193         default:
1194                 PVR_DPF(PVR_DBG_ERROR,
1195                         "%s: Unknown LinuxMemArea type (%d)\n",
1196                         __func__, mem_area->eAreaType);
1197                 BUG();
1198         }
1199 }
1200
1201 IMG_BOOL LinuxMemAreaPhysIsContig(struct LinuxMemArea *psLinuxMemArea)
1202 {
1203         switch (psLinuxMemArea->eAreaType) {
1204         case LINUX_MEM_AREA_IOREMAP:
1205         case LINUX_MEM_AREA_IO:
1206                 return IMG_TRUE;
1207
1208         case LINUX_MEM_AREA_EXTERNAL_KV:
1209                 return psLinuxMemArea->uData.sExternalKV.bPhysContig;
1210
1211         case LINUX_MEM_AREA_VMALLOC:
1212         case LINUX_MEM_AREA_ALLOC_PAGES:
1213                 return IMG_FALSE;
1214
1215         case LINUX_MEM_AREA_SUB_ALLOC:
1216                 return LinuxMemAreaPhysIsContig(psLinuxMemArea->uData.sSubAlloc.
1217                                                 psParentLinuxMemArea);
1218
1219         default:
1220                 PVR_DPF(PVR_DBG_ERROR,
1221                          "%s: Unknown struct LinuxMemArea type (%d)\n",
1222                          __func__, psLinuxMemArea->eAreaType);
1223                 break;
1224         }
1225         return IMG_FALSE;
1226 }
1227
1228 const char *LinuxMemAreaTypeToString(enum LINUX_MEM_AREA_TYPE eMemAreaType)
1229 {
1230         switch (eMemAreaType) {
1231         case LINUX_MEM_AREA_IOREMAP:
1232                 return "LINUX_MEM_AREA_IOREMAP";
1233         case LINUX_MEM_AREA_EXTERNAL_KV:
1234                 return "LINUX_MEM_AREA_EXTERNAL_KV";
1235         case LINUX_MEM_AREA_IO:
1236                 return "LINUX_MEM_AREA_IO";
1237         case LINUX_MEM_AREA_VMALLOC:
1238                 return "LINUX_MEM_AREA_VMALLOC";
1239         case LINUX_MEM_AREA_SUB_ALLOC:
1240                 return "LINUX_MEM_AREA_SUB_ALLOC";
1241         case LINUX_MEM_AREA_ALLOC_PAGES:
1242                 return "LINUX_MEM_AREA_ALLOC_PAGES";
1243         default:
1244                 PVR_ASSERT(0);
1245         }
1246
1247         return "";
1248 }
1249
1250 #if defined(DEBUG_LINUX_MEM_AREAS)
1251 static off_t printLinuxMemAreaRecords(char *buffer, size_t count, off_t off)
1252 {
1253         struct DEBUG_LINUX_MEM_AREA_REC *psRecord;
1254         off_t Ret;
1255
1256         mutex_lock(&g_sDebugMutex);
1257
1258         if (!off) {
1259                 if (count < 500) {
1260                         Ret = 0;
1261                         goto unlock_and_return;
1262                 }
1263                 Ret = printAppend(buffer, count, 0,
1264                                   "Number of Linux Memory Areas: %u\n"
1265                           "At the current water mark these areas "
1266                                   "correspond to %u bytes "
1267                                   "(excluding SUB areas)\n"
1268                           "At the highest water mark these areas "
1269                                   "corresponded to %u bytes "
1270                                   "(excluding SUB areas)\n"
1271                           "\nDetails for all Linux Memory Areas:\n"
1272                           "%s %-24s %s %s %-8s %-5s %s\n",
1273                           g_LinuxMemAreaCount,
1274                           g_LinuxMemAreaWaterMark,
1275                           g_LinuxMemAreaHighWaterMark,
1276                           "psLinuxMemArea",
1277                           "LinuxMemType",
1278                           "CpuVAddr",
1279                           "CpuPAddr", "Bytes", "Pid", "Flags");
1280                 goto unlock_and_return;
1281         }
1282
1283         for (psRecord = g_LinuxMemAreaRecords; --off && psRecord;
1284              psRecord = psRecord->psNext)
1285                 ;
1286         if (!psRecord) {
1287                 Ret = END_OF_FILE;
1288                 goto unlock_and_return;
1289         }
1290
1291         if (count < 500) {
1292                 Ret = 0;
1293                 goto unlock_and_return;
1294         }
1295
1296         Ret = printAppend(buffer, count, 0,
1297                           "%8p       %-24s %8p %08x %-8d %-5u %08x=(%s)\n",
1298                           psRecord->psLinuxMemArea,
1299                           LinuxMemAreaTypeToString(psRecord->psLinuxMemArea->
1300                                                    eAreaType),
1301                           LinuxMemAreaToCpuVAddr(psRecord->psLinuxMemArea),
1302                           LinuxMemAreaToCpuPAddr(psRecord->psLinuxMemArea,
1303                                                  0).uiAddr,
1304                           psRecord->psLinuxMemArea->ui32ByteSize, psRecord->pid,
1305                           psRecord->ui32Flags,
1306                           HAPFlagsToString(psRecord->ui32Flags)
1307             );
1308
1309 unlock_and_return:
1310         mutex_unlock(&g_sDebugMutex);
1311         return Ret;
1312 }
1313 #endif
1314
1315 #if defined(DEBUG_LINUX_MEMORY_ALLOCATIONS)
1316 static off_t printMemoryRecords(char *buffer, size_t count, off_t off)
1317 {
1318         struct DEBUG_MEM_ALLOC_REC *psRecord;
1319         off_t Ret;
1320
1321         mutex_lock(&g_sDebugMutex);
1322
1323         if (!off) {
1324                 if (count < 1000) {
1325                         Ret = 0;
1326                         goto unlock_and_return;
1327                 }
1328
1329                 Ret = printAppend(buffer, count, 0, "%-60s: %d bytes\n",
1330                           "Current Water Mark of bytes allocated via kmalloc",
1331                                  g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_KMALLOC]);
1332                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1333                         "Highest Water Mark of bytes allocated via kmalloc",
1334                                 g_HighWaterMarkData
1335                                 [DEBUG_MEM_ALLOC_TYPE_KMALLOC]);
1336                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1337                         "Current Water Mark of bytes allocated via vmalloc",
1338                         g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_VMALLOC]);
1339                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1340                         "Highest Water Mark of bytes allocated via vmalloc",
1341                                 g_HighWaterMarkData
1342                                 [DEBUG_MEM_ALLOC_TYPE_VMALLOC]);
1343                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1344                         "Current Water Mark of bytes allocated via alloc_pages",
1345                                 g_WaterMarkData
1346                                 [DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]);
1347                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1348                         "Highest Water Mark of bytes allocated via alloc_pages",
1349                                 g_HighWaterMarkData
1350                                 [DEBUG_MEM_ALLOC_TYPE_ALLOC_PAGES]);
1351                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1352                         "Current Water Mark of bytes allocated via ioremap",
1353                 g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IOREMAP]);
1354                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1355                         "Highest Water Mark of bytes allocated via ioremap",
1356                                 g_HighWaterMarkData
1357                                 [DEBUG_MEM_ALLOC_TYPE_IOREMAP]);
1358                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1359                         "Current Water Mark of bytes reserved for "
1360                         "\"IO\" memory areas",
1361                 g_WaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]);
1362                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1363                         "Highest Water Mark of bytes allocated for "
1364                         "\"IO\" memory areas",
1365                 g_HighWaterMarkData[DEBUG_MEM_ALLOC_TYPE_IO]);
1366                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1367                         "Current Water Mark of bytes allocated via "
1368                         "kmem_cache_alloc",
1369                                 g_WaterMarkData
1370                                 [DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]);
1371                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1372                         "Highest Water Mark of bytes allocated via "
1373                         "kmem_cache_alloc",
1374                                 g_HighWaterMarkData
1375                                 [DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE]);
1376                 Ret = printAppend(buffer, count, Ret, "\n");
1377
1378                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1379                   "The Current Water Mark for memory allocated from system RAM",
1380                   g_SysRAMWaterMark);
1381                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1382                   "The Highest Water Mark for memory allocated from system RAM",
1383                   g_SysRAMHighWaterMark);
1384                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1385                   "The Current Water Mark for memory allocated from IO memory",
1386                   g_IOMemWaterMark);
1387                 Ret = printAppend(buffer, count, Ret, "%-60s: %d bytes\n",
1388                   "The Highest Water Mark for memory allocated from IO memory",
1389                   g_IOMemHighWaterMark);
1390
1391                 Ret = printAppend(buffer, count, Ret, "\n");
1392
1393                 Ret = printAppend(buffer, count, Ret,
1394                                 "Details for all known allocations:\n"
1395                                 "%-16s %-8s %-8s %-10s %-5s %-10s %s\n", "Type",
1396                                 "CpuVAddr", "CpuPAddr", "Bytes", "PID",
1397                                 "PrivateData", "Filename:Line");
1398
1399
1400                 goto unlock_and_return;
1401         }
1402
1403         if (count < 1000) {
1404                 Ret = 0;
1405                 goto unlock_and_return;
1406         }
1407
1408         for (psRecord = g_MemoryRecords; --off && psRecord;
1409              psRecord = psRecord->psNext)
1410                 ;
1411         if (!psRecord) {
1412                 Ret = END_OF_FILE;
1413                 goto unlock_and_return;
1414         }
1415
1416         if (psRecord->eAllocType != DEBUG_MEM_ALLOC_TYPE_KMEM_CACHE)
1417                 Ret = printAppend(buffer, count, 0,
1418                          "%-16s %-8p %08x %-10d %-5d %-10s %s:%d\n",
1419                          DebugMemAllocRecordTypeToString(psRecord->eAllocType),
1420                          psRecord->pvCpuVAddr, psRecord->ulCpuPAddr,
1421                          psRecord->ui32Bytes, psRecord->pid, "NULL",
1422                          psRecord->pszFileName, psRecord->ui32Line);
1423         else
1424                 Ret = printAppend(buffer, count, 0,
1425                           "%-16s %-8p %08x %-10d %-5d %-10s %s:%d\n",
1426                           DebugMemAllocRecordTypeToString(psRecord->eAllocType),
1427                           psRecord->pvCpuVAddr, psRecord->ulCpuPAddr,
1428                           psRecord->ui32Bytes, psRecord->pid,
1429                           KMemCacheNameWrapper(psRecord->pvPrivateData),
1430                           psRecord->pszFileName, psRecord->ui32Line);
1431
1432 unlock_and_return:
1433         mutex_unlock(&g_sDebugMutex);
1434         return Ret;
1435 }
1436 #endif
1437
1438 #if defined(DEBUG_LINUX_MEM_AREAS) || defined(DEBUG_LINUX_MMAP_AREAS)
1439 const char *HAPFlagsToString(u32 ui32Flags)
1440 {
1441         static char szFlags[50];
1442         s32 i32Pos = 0;
1443         u32 ui32CacheTypeIndex, ui32MapTypeIndex;
1444         char *apszCacheTypes[] = {
1445                 "UNCACHED",
1446                 "CACHED",
1447                 "WRITECOMBINE",
1448                 "UNKNOWN"
1449         };
1450         char *apszMapType[] = {
1451                 "KERNEL_ONLY",
1452                 "SINGLE_PROCESS",
1453                 "MULTI_PROCESS",
1454                 "FROM_EXISTING_PROCESS",
1455                 "NO_CPU_VIRTUAL",
1456                 "UNKNOWN"
1457         };
1458
1459         if (ui32Flags & PVRSRV_HAP_UNCACHED) {
1460                 ui32CacheTypeIndex = 0;
1461         } else if (ui32Flags & PVRSRV_HAP_CACHED) {
1462                 ui32CacheTypeIndex = 1;
1463         } else if (ui32Flags & PVRSRV_HAP_WRITECOMBINE) {
1464                 ui32CacheTypeIndex = 2;
1465         } else {
1466                 ui32CacheTypeIndex = 3;
1467                 PVR_DPF(PVR_DBG_ERROR, "%s: unknown cache type (%u)",
1468                          __func__, (ui32Flags & PVRSRV_HAP_CACHETYPE_MASK));
1469         }
1470
1471         if (ui32Flags & PVRSRV_HAP_KERNEL_ONLY) {
1472                 ui32MapTypeIndex = 0;
1473         } else if (ui32Flags & PVRSRV_HAP_SINGLE_PROCESS) {
1474                 ui32MapTypeIndex = 1;
1475         } else if (ui32Flags & PVRSRV_HAP_MULTI_PROCESS) {
1476                 ui32MapTypeIndex = 2;
1477         } else if (ui32Flags & PVRSRV_HAP_FROM_EXISTING_PROCESS) {
1478                 ui32MapTypeIndex = 3;
1479         } else if (ui32Flags & PVRSRV_HAP_NO_CPU_VIRTUAL) {
1480                 ui32MapTypeIndex = 4;
1481         } else {
1482                 ui32MapTypeIndex = 5;
1483                 PVR_DPF(PVR_DBG_ERROR, "%s: unknown map type (%u)",
1484                          __func__, (ui32Flags & PVRSRV_HAP_MAPTYPE_MASK));
1485         }
1486
1487         i32Pos = sprintf(szFlags, "%s|", apszCacheTypes[ui32CacheTypeIndex]);
1488         if (i32Pos <= 0) {
1489                 PVR_DPF(PVR_DBG_ERROR,
1490                          "%s: sprintf for cache type %u failed (%d)", __func__,
1491                          ui32CacheTypeIndex, i32Pos);
1492                 szFlags[0] = 0;
1493         } else {
1494                 sprintf(szFlags + i32Pos, "%s", apszMapType[ui32MapTypeIndex]);
1495         }
1496
1497         return szFlags;
1498 }
1499 #endif