pvr/buffer_manager.c

   1 /**********************************************************************
   2  *
   3  * Copyright(c) 2008 Imagination Technologies Ltd. All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful but, except
  10  * as otherwise stated in writing, without any warranty; without even the
  11  * implied warranty of merchantability or fitness for a particular purpose.
  12  * See the GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * Imagination Technologies Ltd. <gpl-support@imgtec.com>
  23  * Home Park Estate, Kings Langley, Herts, WD4 8LZ, UK
  24  *
  25  ******************************************************************************/
  26 #include "services_headers.h"
  27
  28 #include "sysconfig.h"
  29 #include "hash.h"
  30 #include "ra.h"
  31 #include "pvr_pdump.h"
  32 #include "mmu.h"
  33
  34 #define MIN(a, b)       (a > b ? b : a)
  35
  36 static IMG_BOOL ZeroBuf(struct BM_BUF *pBuf, struct BM_MAPPING *pMapping,
  37                 u32 ui32Bytes, u32 ui32Flags);
  38 static void BM_FreeMemory(void *pH, u32 base, struct BM_MAPPING *psMapping);
  39 static IMG_BOOL BM_ImportMemory(void *pH, size_t uSize,
  40                 size_t *pActualSize, struct BM_MAPPING **ppsMapping, u32 uFlags,
  41                 u32 *pBase);
  42
  43 static IMG_BOOL DevMemoryAlloc(struct BM_CONTEXT *pBMContext,
  44                 struct BM_MAPPING *pMapping, u32 uFlags,
  45                 u32 dev_vaddr_alignment, struct IMG_DEV_VIRTADDR *pDevVAddr);
  46 static void DevMemoryFree(struct BM_MAPPING *pMapping);
  47
  48 static IMG_BOOL AllocMemory(struct BM_CONTEXT *pBMContext,
  49                 struct BM_HEAP *psBMHeap, struct IMG_DEV_VIRTADDR *psDevVAddr,
  50                 size_t uSize, u32 uFlags, u32 uDevVAddrAlignment,
  51                 struct BM_BUF *pBuf)
  52 {
  53         struct BM_MAPPING *pMapping;
  54         u32 uOffset;
  55         struct RA_ARENA *pArena = NULL;
  56
  57         PVR_DPF(PVR_DBG_MESSAGE, "AllocMemory "
  58                         "(pBMContext=%08X, uSize=0x%x, uFlags=0x%x, "
  59                         "align=0x%x, pBuf=%08X)",
  60                  pBMContext, uSize, uFlags, uDevVAddrAlignment, pBuf);
  61
  62         if (uFlags & PVRSRV_MEM_RAM_BACKED_ALLOCATION) {
  63                 if (uFlags & PVRSRV_MEM_USER_SUPPLIED_DEVVADDR) {
  64                         PVR_DPF(PVR_DBG_ERROR, "AllocMemory: "
  65                                 "combination of DevVAddr management and "
  66                                 "RAM backing mode unsupported");
  67                         return IMG_FALSE;
  68                 }
  69
  70                 if (psBMHeap->ui32Attribs &
  71                     (PVRSRV_BACKINGSTORE_SYSMEM_NONCONTIG |
  72                      PVRSRV_BACKINGSTORE_LOCALMEM_CONTIG)) {
  73                         pArena = psBMHeap->pImportArena;
  74                 } else {
  75                         PVR_DPF(PVR_DBG_ERROR, "AllocMemory: "
  76                                 "backing store type doesn't match heap");
  77                         return IMG_FALSE;
  78                 }
  79
  80                 if (!RA_Alloc(pArena, uSize, (void *)&pMapping, uFlags,
  81                               uDevVAddrAlignment,
  82                               (u32 *)&(pBuf->DevVAddr.uiAddr))) {
  83                         PVR_DPF(PVR_DBG_ERROR,
  84                                  "AllocMemory: RA_Alloc(0x%x) FAILED", uSize);
  85                         return IMG_FALSE;
  86                 }
  87
  88                 uOffset = pBuf->DevVAddr.uiAddr - pMapping->DevVAddr.uiAddr;
  89                 if (pMapping->CpuVAddr) {
  90                         pBuf->CpuVAddr =
  91                             (void *)((u32) pMapping->CpuVAddr + uOffset);
  92                 } else {
  93                         pBuf->CpuVAddr = NULL;
  94                 }
  95
  96                 if (uSize == pMapping->uSize) {
  97                         pBuf->hOSMemHandle = pMapping->hOSMemHandle;
  98                 } else {
  99                         if (OSGetSubMemHandle(pMapping->hOSMemHandle, uOffset,
 100                                         uSize, psBMHeap->ui32Attribs,
 101                                         &pBuf->hOSMemHandle) != PVRSRV_OK) {
 102                                 PVR_DPF(PVR_DBG_ERROR, "AllocMemory: "
 103                                                 "OSGetSubMemHandle FAILED");
 104                                 return IMG_FALSE;
 105                         }
 106                 }
 107
 108                 pBuf->CpuPAddr.uiAddr = pMapping->CpuPAddr.uiAddr + uOffset;
 109
 110                 if (uFlags & PVRSRV_MEM_ZERO)
 111                         if (!ZeroBuf(pBuf, pMapping, uSize,
 112                                      psBMHeap->ui32Attribs | uFlags))
 113                                 return IMG_FALSE;
 114         } else {
 115                 if (uFlags & PVRSRV_MEM_USER_SUPPLIED_DEVVADDR) {
 116                         PVR_ASSERT(psDevVAddr != NULL);
 117
 118                         if (psDevVAddr == NULL) {
 119                                 PVR_DPF(PVR_DBG_ERROR, "AllocMemory: "
 120                                         "invalid parameter - psDevVAddr");
 121                                 return IMG_FALSE;
 122                         }
 123
 124                         pBMContext->psDeviceNode->pfnMMUAlloc(
 125                                               psBMHeap->pMMUHeap, uSize,
 126                                               PVRSRV_MEM_USER_SUPPLIED_DEVVADDR,
 127                                               uDevVAddrAlignment, psDevVAddr);
 128                         pBuf->DevVAddr = *psDevVAddr;
 129                 } else {
 130                         pBMContext->psDeviceNode->pfnMMUAlloc(psBMHeap->
 131                                                              pMMUHeap, uSize, 0,
 132                                                              uDevVAddrAlignment,
 133                                                              &pBuf->DevVAddr);
 134                 }
 135
 136                 if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP,
 137                                sizeof(struct BM_MAPPING),
 138                                (void **)&pMapping, NULL) != PVRSRV_OK) {
 139                         PVR_DPF(PVR_DBG_ERROR,
 140                                  "AllocMemory: OSAllocMem(0x%x) FAILED");
 141                         return IMG_FALSE;
 142                 }
 143
 144                 pBuf->CpuVAddr = NULL;
 145                 pBuf->hOSMemHandle = NULL;
 146                 pBuf->CpuPAddr.uiAddr = 0;
 147
 148                 pMapping->CpuVAddr = NULL;
 149                 pMapping->CpuPAddr.uiAddr = 0;
 150                 pMapping->DevVAddr = pBuf->DevVAddr;
 151                 pMapping->psSysAddr = NULL;
 152                 pMapping->uSize = uSize;
 153                 pMapping->hOSMemHandle = NULL;
 154         }
 155
 156         pMapping->pArena = pArena;
 157
 158         pMapping->pBMHeap = psBMHeap;
 159         pBuf->pMapping = pMapping;
 160
 161         PVR_DPF(PVR_DBG_MESSAGE, "AllocMemory: "
 162                 "pMapping=%08X: DevV=%08X CpuV=%08X CpuP=%08X uSize=0x%x",
 163                  pMapping, pMapping->DevVAddr.uiAddr, pMapping->CpuVAddr,
 164                  pMapping->CpuPAddr.uiAddr, pMapping->uSize);
 165
 166         PVR_DPF(PVR_DBG_MESSAGE, "AllocMemory: "
 167                  "pBuf=%08X: DevV=%08X CpuV=%08X CpuP=%08X uSize=0x%x",
 168                  pBuf, pBuf->DevVAddr.uiAddr, pBuf->CpuVAddr,
 169                  pBuf->CpuPAddr.uiAddr, uSize);
 170
 171         PVR_ASSERT(((pBuf->DevVAddr.uiAddr) & (uDevVAddrAlignment - 1)) == 0);
 172
 173         return IMG_TRUE;
 174 }
 175
 176 static IMG_BOOL WrapMemory(struct BM_HEAP *psBMHeap,
 177            size_t uSize, u32 ui32BaseOffset, IMG_BOOL bPhysContig,
 178            struct IMG_SYS_PHYADDR *psAddr, void *pvCPUVAddr, u32 uFlags,
 179            struct BM_BUF *pBuf)
 180 {
 181         struct IMG_DEV_VIRTADDR DevVAddr = { 0 };
 182         struct BM_MAPPING *pMapping;
 183         IMG_BOOL bResult;
 184         u32 const ui32PageSize = HOST_PAGESIZE();
 185
 186         PVR_DPF(PVR_DBG_MESSAGE,
 187                  "WrapMemory(psBMHeap=%08X, size=0x%x, offset=0x%x, "
 188                  "bPhysContig=0x%x, pvCPUVAddr = 0x%x, flags=0x%x, pBuf=%08X)",
 189                  psBMHeap, uSize, ui32BaseOffset, bPhysContig, pvCPUVAddr,
 190                  uFlags, pBuf);
 191
 192         PVR_ASSERT((psAddr->uiAddr & (ui32PageSize - 1)) == 0);
 193
 194         PVR_ASSERT(((u32) pvCPUVAddr & (ui32PageSize - 1)) == 0);
 195
 196         uSize += ui32BaseOffset;
 197         uSize = HOST_PAGEALIGN(uSize);
 198
 199         if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(*pMapping),
 200                        (void **)&pMapping, NULL) != PVRSRV_OK) {
 201                 PVR_DPF(PVR_DBG_ERROR, "WrapMemory: OSAllocMem(0x%x) FAILED",
 202                          sizeof(*pMapping));
 203                 return IMG_FALSE;
 204         }
 205
 206         OSMemSet(pMapping, 0, sizeof(*pMapping));
 207
 208         pMapping->uSize = uSize;
 209         pMapping->pBMHeap = psBMHeap;
 210
 211         if (pvCPUVAddr) {
 212                 pMapping->CpuVAddr = pvCPUVAddr;
 213
 214                 if (bPhysContig) {
 215                         pMapping->eCpuMemoryOrigin = hm_wrapped_virtaddr;
 216                         pMapping->CpuPAddr = SysSysPAddrToCpuPAddr(psAddr[0]);
 217
 218                         if (OSRegisterMem(pMapping->CpuPAddr,
 219                                         pMapping->CpuVAddr, pMapping->uSize,
 220                                           uFlags, &pMapping->hOSMemHandle) !=
 221                                         PVRSRV_OK) {
 222                                 PVR_DPF(PVR_DBG_ERROR, "WrapMemory: "
 223                                         "OSRegisterMem Phys=0x%08X, "
 224                                         "CpuVAddr = 0x%08X, Size=%d) failed",
 225                                          pMapping->CpuPAddr, pMapping->CpuVAddr,
 226                                          pMapping->uSize);
 227                                 goto fail_cleanup;
 228                         }
 229                 } else {
 230                         pMapping->eCpuMemoryOrigin =
 231                             hm_wrapped_scatter_virtaddr;
 232                         pMapping->psSysAddr = psAddr;
 233
 234                         if (OSRegisterDiscontigMem(pMapping->psSysAddr,
 235                                                    pMapping->CpuVAddr,
 236                                                    pMapping->uSize,
 237                                                    uFlags,
 238                                                    &pMapping->hOSMemHandle) !=
 239                             PVRSRV_OK) {
 240                                 PVR_DPF(PVR_DBG_ERROR, "WrapMemory: "
 241                                         "OSRegisterDiscontigMem CpuVAddr = "
 242                                         "0x%08X, Size=%d) failed",
 243                                          pMapping->CpuVAddr, pMapping->uSize);
 244                                 goto fail_cleanup;
 245                         }
 246                 }
 247         } else {
 248                 if (bPhysContig) {
 249                         pMapping->eCpuMemoryOrigin = hm_wrapped;
 250                         pMapping->CpuPAddr = SysSysPAddrToCpuPAddr(psAddr[0]);
 251
 252                         if (OSReservePhys(pMapping->CpuPAddr, pMapping->uSize,
 253                                         uFlags, &pMapping->CpuVAddr,
 254                                         &pMapping->hOSMemHandle) != PVRSRV_OK) {
 255                                 PVR_DPF(PVR_DBG_ERROR, "WrapMemory: "
 256                                         "OSReservePhys Phys=0x%08X, Size=%d) "
 257                                         "failed",
 258                                          pMapping->CpuPAddr, pMapping->uSize);
 259                                 goto fail_cleanup;
 260                         }
 261                 } else {
 262                         pMapping->eCpuMemoryOrigin = hm_wrapped_scatter;
 263                         pMapping->psSysAddr = psAddr;
 264
 265                         if (OSReserveDiscontigPhys(pMapping->psSysAddr,
 266                                                    pMapping->uSize, uFlags,
 267                                                    &pMapping->CpuVAddr,
 268                                                    &pMapping->hOSMemHandle) !=
 269                             PVRSRV_OK) {
 270                                 PVR_DPF(PVR_DBG_ERROR, "WrapMemory: "
 271                                        "OSReserveDiscontigPhys Size=%d) failed",
 272                                          pMapping->uSize);
 273                                 goto fail_cleanup;
 274                         }
 275                 }
 276         }
 277
 278         bResult = DevMemoryAlloc(psBMHeap->pBMContext, pMapping,
 279                                  uFlags | PVRSRV_MEM_READ | PVRSRV_MEM_WRITE,
 280                                  ui32PageSize, &DevVAddr);
 281         if (!bResult) {
 282                 PVR_DPF(PVR_DBG_ERROR,
 283                          "WrapMemory: DevMemoryAlloc(0x%x) failed",
 284                          pMapping->uSize);
 285                 goto fail_cleanup;
 286         }
 287
 288         pBuf->CpuPAddr.uiAddr = pMapping->CpuPAddr.uiAddr + ui32BaseOffset;
 289         if (!ui32BaseOffset)
 290                 pBuf->hOSMemHandle = pMapping->hOSMemHandle;
 291         else
 292                 if (OSGetSubMemHandle(pMapping->hOSMemHandle,
 293                                       ui32BaseOffset,
 294                                       (pMapping->uSize - ui32BaseOffset),
 295                                       uFlags,
 296                                       &pBuf->hOSMemHandle) != PVRSRV_OK) {
 297                         PVR_DPF(PVR_DBG_ERROR,
 298                                  "WrapMemory: OSGetSubMemHandle failed");
 299                         goto fail_cleanup;
 300                 }
 301         if (pMapping->CpuVAddr)
 302                 pBuf->CpuVAddr = (void *)((u32) pMapping->CpuVAddr +
 303                                                              ui32BaseOffset);
 304         pBuf->DevVAddr.uiAddr = pMapping->DevVAddr.uiAddr + ui32BaseOffset;
 305
 306         if (uFlags & PVRSRV_MEM_ZERO)
 307                 if (!ZeroBuf(pBuf, pMapping, uSize, uFlags))
 308                         return IMG_FALSE;
 309
 310         PVR_DPF(PVR_DBG_MESSAGE, "DevVaddr.uiAddr=%08X", DevVAddr.uiAddr);
 311         PVR_DPF(PVR_DBG_MESSAGE, "WrapMemory: pMapping=%08X: DevV=%08X "
 312                         "CpuV=%08X CpuP=%08X uSize=0x%x",
 313                  pMapping, pMapping->DevVAddr.uiAddr, pMapping->CpuVAddr,
 314                  pMapping->CpuPAddr.uiAddr, pMapping->uSize);
 315         PVR_DPF(PVR_DBG_MESSAGE, "WrapMemory: pBuf=%08X: DevV=%08X "
 316                                 "CpuV=%08X CpuP=%08X uSize=0x%x",
 317                  pBuf, pBuf->DevVAddr.uiAddr, pBuf->CpuVAddr,
 318                  pBuf->CpuPAddr.uiAddr, uSize);
 319
 320         pBuf->pMapping = pMapping;
 321         return IMG_TRUE;
 322
 323 fail_cleanup:
 324         if (ui32BaseOffset && pBuf->hOSMemHandle)
 325                 OSReleaseSubMemHandle(pBuf->hOSMemHandle, uFlags);
 326
 327         if (pMapping->CpuVAddr || pMapping->hOSMemHandle)
 328                 switch (pMapping->eCpuMemoryOrigin) {
 329                 case hm_wrapped:
 330                         OSUnReservePhys(pMapping->CpuVAddr, pMapping->uSize,
 331                                         uFlags, pMapping->hOSMemHandle);
 332                         break;
 333                 case hm_wrapped_virtaddr:
 334                         OSUnRegisterMem(pMapping->CpuVAddr, pMapping->uSize,
 335                                         uFlags, pMapping->hOSMemHandle);
 336                         break;
 337                 case hm_wrapped_scatter:
 338                         OSUnReserveDiscontigPhys(pMapping->CpuVAddr,
 339                                                  pMapping->uSize, uFlags,
 340                                                  pMapping->hOSMemHandle);
 341                         break;
 342                 case hm_wrapped_scatter_virtaddr:
 343                         OSUnRegisterDiscontigMem(pMapping->CpuVAddr,
 344                                                  pMapping->uSize, uFlags,
 345                                                  pMapping->hOSMemHandle);
 346                         break;
 347                 default:
 348                         break;
 349                 }
 350
 351         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_MAPPING), pMapping,
 352                   NULL);
 353
 354         return IMG_FALSE;
 355 }
 356
 357 static IMG_BOOL ZeroBuf(struct BM_BUF *pBuf, struct BM_MAPPING *pMapping,
 358                         u32 ui32Bytes, u32 ui32Flags)
 359 {
 360         void *pvCpuVAddr;
 361
 362         if (pBuf->CpuVAddr) {
 363                 OSMemSet(pBuf->CpuVAddr, 0, ui32Bytes);
 364         } else if (pMapping->eCpuMemoryOrigin == hm_contiguous ||
 365                    pMapping->eCpuMemoryOrigin == hm_wrapped) {
 366                 pvCpuVAddr = (void __force *)OSMapPhysToLin(pBuf->CpuPAddr,
 367                                             ui32Bytes,
 368                                             PVRSRV_HAP_KERNEL_ONLY |
 369                                             (ui32Flags &
 370                                                PVRSRV_HAP_CACHETYPE_MASK),
 371                                             NULL);
 372                 if (!pvCpuVAddr) {
 373                         PVR_DPF(PVR_DBG_ERROR, "ZeroBuf: "
 374                                 "OSMapPhysToLin for contiguous buffer failed");
 375                         return IMG_FALSE;
 376                 }
 377                 OSMemSet(pvCpuVAddr, 0, ui32Bytes);
 378                 OSUnMapPhysToLin((void __force __iomem *)pvCpuVAddr, ui32Bytes,
 379                                  PVRSRV_HAP_KERNEL_ONLY |
 380                                     (ui32Flags & PVRSRV_HAP_CACHETYPE_MASK),
 381                                  NULL);
 382         } else {
 383                 u32 ui32BytesRemaining = ui32Bytes;
 384                 u32 ui32CurrentOffset = 0;
 385                 struct IMG_CPU_PHYADDR CpuPAddr;
 386
 387                 PVR_ASSERT(pBuf->hOSMemHandle);
 388
 389                 while (ui32BytesRemaining > 0) {
 390                         u32 ui32BlockBytes =
 391                             MIN(ui32BytesRemaining, HOST_PAGESIZE());
 392                         CpuPAddr =
 393                             OSMemHandleToCpuPAddr(pBuf->hOSMemHandle,
 394                                                   ui32CurrentOffset);
 395
 396                         if (CpuPAddr.uiAddr & (HOST_PAGESIZE() - 1))
 397                                 ui32BlockBytes =
 398                                     MIN(ui32BytesRemaining,
 399                                         HOST_PAGEALIGN(CpuPAddr.uiAddr) -
 400                                         CpuPAddr.uiAddr);
 401
 402                         pvCpuVAddr = (void __force *)OSMapPhysToLin(CpuPAddr,
 403                                                     ui32BlockBytes,
 404                                                     PVRSRV_HAP_KERNEL_ONLY |
 405                                                      (ui32Flags &
 406                                                      PVRSRV_HAP_CACHETYPE_MASK),
 407                                                     NULL);
 408                         if (!pvCpuVAddr) {
 409                                 PVR_DPF(PVR_DBG_ERROR, "ZeroBuf: "
 410                                         "OSMapPhysToLin while "
 411                                        "zeroing non-contiguous memory FAILED");
 412                                 return IMG_FALSE;
 413                         }
 414                         OSMemSet(pvCpuVAddr, 0, ui32BlockBytes);
 415                         OSUnMapPhysToLin((void __force __iomem *)pvCpuVAddr,
 416                                          ui32BlockBytes,
 417                                          PVRSRV_HAP_KERNEL_ONLY |
 418                                            (ui32Flags &
 419                                             PVRSRV_HAP_CACHETYPE_MASK),
 420                                          NULL);
 421
 422                         ui32BytesRemaining -= ui32BlockBytes;
 423                         ui32CurrentOffset += ui32BlockBytes;
 424                 }
 425         }
 426
 427         return IMG_TRUE;
 428 }
 429
 430 static void FreeBuf(struct BM_BUF *pBuf, u32 ui32Flags)
 431 {
 432         struct BM_MAPPING *pMapping;
 433
 434         PVR_DPF(PVR_DBG_MESSAGE,
 435                 "FreeBuf: pBuf=%08X: DevVAddr=%08X CpuVAddr=%08X CpuPAddr=%08X",
 436                  pBuf, pBuf->DevVAddr.uiAddr, pBuf->CpuVAddr,
 437                  pBuf->CpuPAddr.uiAddr);
 438
 439         pMapping = pBuf->pMapping;
 440
 441         if (ui32Flags & PVRSRV_MEM_USER_SUPPLIED_DEVVADDR) {
 442                 if (ui32Flags & PVRSRV_MEM_RAM_BACKED_ALLOCATION)
 443                         PVR_DPF(PVR_DBG_ERROR, "FreeBuf: "
 444                                 "combination of DevVAddr management "
 445                                 "and RAM backing mode unsupported");
 446                 else
 447                         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP,
 448                                         sizeof(struct BM_MAPPING),
 449                                   pMapping, NULL);
 450         } else {
 451                 if (pBuf->hOSMemHandle != pMapping->hOSMemHandle)
 452                         OSReleaseSubMemHandle(pBuf->hOSMemHandle, ui32Flags);
 453                 if (ui32Flags & PVRSRV_MEM_RAM_BACKED_ALLOCATION) {
 454                         RA_Free(pBuf->pMapping->pArena, pBuf->DevVAddr.uiAddr,
 455                                 IMG_FALSE);
 456                 } else {
 457                         switch (pMapping->eCpuMemoryOrigin) {
 458                         case hm_wrapped:
 459                                 OSUnReservePhys(pMapping->CpuVAddr,
 460                                                 pMapping->uSize, ui32Flags,
 461                                                 pMapping->hOSMemHandle);
 462                                 break;
 463                         case hm_wrapped_virtaddr:
 464                                 OSUnRegisterMem(pMapping->CpuVAddr,
 465                                                 pMapping->uSize, ui32Flags,
 466                                                 pMapping->hOSMemHandle);
 467                                 break;
 468                         case hm_wrapped_scatter:
 469                                 OSUnReserveDiscontigPhys(pMapping->CpuVAddr,
 470                                                          pMapping->uSize,
 471                                                          ui32Flags,
 472                                                          pMapping->
 473                                                          hOSMemHandle);
 474                                 break;
 475                         case hm_wrapped_scatter_virtaddr:
 476                                 OSUnRegisterDiscontigMem(pMapping->CpuVAddr,
 477                                                          pMapping->uSize,
 478                                                          ui32Flags,
 479                                                          pMapping->
 480                                                          hOSMemHandle);
 481                                 break;
 482                         default:
 483                                 break;
 484                         }
 485
 486                         DevMemoryFree(pMapping);
 487
 488                         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP,
 489                                   sizeof(struct BM_MAPPING), pMapping, NULL);
 490                 }
 491         }
 492
 493         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_BUF), pBuf, NULL);
 494 }
 495
 496 void BM_DestroyContext(void *hBMContext)
 497 {
 498         struct BM_CONTEXT *pBMContext = (struct BM_CONTEXT *)hBMContext;
 499         struct BM_HEAP *psBMHeap;
 500
 501         PVR_DPF(PVR_DBG_MESSAGE, "BM_DestroyContext");
 502
 503         for (psBMHeap = pBMContext->psBMHeap;
 504              psBMHeap != NULL; psBMHeap = psBMHeap->psNext)
 505                 if (psBMHeap->ui32Attribs &
 506                     (PVRSRV_BACKINGSTORE_SYSMEM_NONCONTIG |
 507                       PVRSRV_BACKINGSTORE_LOCALMEM_CONTIG))
 508                         if (psBMHeap->pImportArena) {
 509                                 IMG_BOOL bTestDelete =
 510                                     RA_TestDelete(psBMHeap->pImportArena);
 511                                 BUG_ON(!bTestDelete);
 512                         }
 513
 514         ResManFreeResByPtr(pBMContext->hResItem);
 515 }
 516
 517 static enum PVRSRV_ERROR BM_DestroyContextCallBack(void *pvParam, u32 ui32Param)
 518 {
 519         struct BM_CONTEXT *pBMContext = pvParam;
 520         struct BM_CONTEXT **ppBMContext;
 521         struct BM_HEAP *psBMHeap, *psTmpBMHeap;
 522         struct PVRSRV_DEVICE_NODE *psDeviceNode;
 523
 524         PVR_UNREFERENCED_PARAMETER(ui32Param);
 525
 526         psDeviceNode = pBMContext->psDeviceNode;
 527
 528         psBMHeap = pBMContext->psBMHeap;
 529         while (psBMHeap) {
 530                 if (psBMHeap->ui32Attribs &
 531                     (PVRSRV_BACKINGSTORE_SYSMEM_NONCONTIG |
 532                       PVRSRV_BACKINGSTORE_LOCALMEM_CONTIG)) {
 533                         if (psBMHeap->pImportArena)
 534                                 RA_Delete(psBMHeap->pImportArena);
 535                 } else {
 536                         PVR_DPF(PVR_DBG_ERROR, "BM_DestroyContext: "
 537                                            "backing store type unsupported");
 538                         return PVRSRV_ERROR_GENERIC;
 539                 }
 540
 541                 psDeviceNode->pfnMMUDelete(psBMHeap->pMMUHeap);
 542
 543                 psTmpBMHeap = psBMHeap;
 544
 545                 psBMHeap = psBMHeap->psNext;
 546
 547                 OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_HEAP),
 548                           psTmpBMHeap, NULL);
 549         }
 550
 551         if (pBMContext->psMMUContext)
 552                 psDeviceNode->pfnMMUFinalise(pBMContext->psMMUContext);
 553
 554         if (pBMContext->pBufferHash)
 555                 HASH_Delete(pBMContext->pBufferHash);
 556
 557         if (pBMContext == psDeviceNode->sDevMemoryInfo.pBMKernelContext) {
 558                 psDeviceNode->sDevMemoryInfo.pBMKernelContext = NULL;
 559         } else {
 560                 for (ppBMContext = &psDeviceNode->sDevMemoryInfo.pBMContext;
 561                      *ppBMContext; ppBMContext = &((*ppBMContext)->psNext))
 562                         if (*ppBMContext == pBMContext) {
 563                                 *ppBMContext = pBMContext->psNext;
 564                                 break;
 565                         }
 566         }
 567
 568         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_CONTEXT),
 569                   pBMContext, NULL);
 570
 571         return PVRSRV_OK;
 572 }
 573
 574 void *BM_CreateContext(struct PVRSRV_DEVICE_NODE *psDeviceNode,
 575                  struct IMG_DEV_PHYADDR *psPDDevPAddr,
 576                  struct PVRSRV_PER_PROCESS_DATA *psPerProc, IMG_BOOL *pbCreated)
 577 {
 578         struct BM_CONTEXT *pBMContext;
 579         struct BM_HEAP *psBMHeap;
 580         struct DEVICE_MEMORY_INFO *psDevMemoryInfo;
 581         IMG_BOOL bKernelContext;
 582         struct RESMAN_CONTEXT *hResManContext;
 583
 584         PVR_DPF(PVR_DBG_MESSAGE, "BM_CreateContext");
 585
 586         if (psPerProc == NULL) {
 587                 bKernelContext = IMG_TRUE;
 588                 hResManContext = psDeviceNode->hResManContext;
 589         } else {
 590                 bKernelContext = IMG_FALSE;
 591                 hResManContext = psPerProc->hResManContext;
 592         }
 593
 594         if (pbCreated != NULL)
 595                 *pbCreated = IMG_FALSE;
 596
 597         psDevMemoryInfo = &psDeviceNode->sDevMemoryInfo;
 598
 599         if (bKernelContext == IMG_FALSE)
 600                 for (pBMContext = psDevMemoryInfo->pBMContext;
 601                      pBMContext != NULL; pBMContext = pBMContext->psNext)
 602                         if (ResManFindResourceByPtr(hResManContext,
 603                                                      pBMContext->hResItem) ==
 604                             PVRSRV_OK) {
 605                                 pBMContext->ui32RefCount++;
 606                                 return (void *)pBMContext;
 607                         }
 608         if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_CONTEXT),
 609                        (void **)&pBMContext, NULL) != PVRSRV_OK) {
 610                 PVR_DPF(PVR_DBG_ERROR, "BM_CreateContext: Alloc failed");
 611                 return NULL;
 612         }
 613         OSMemSet(pBMContext, 0, sizeof(struct BM_CONTEXT));
 614
 615         pBMContext->psDeviceNode = psDeviceNode;
 616
 617         pBMContext->pBufferHash = HASH_Create(32);
 618         if (pBMContext->pBufferHash == NULL) {
 619                 PVR_DPF(PVR_DBG_ERROR,
 620                          "BM_CreateContext: HASH_Create failed");
 621                 goto cleanup;
 622         }
 623
 624         if (psDeviceNode->pfnMMUInitialise(psDeviceNode,
 625                                            &pBMContext->psMMUContext,
 626                                            psPDDevPAddr) != PVRSRV_OK) {
 627                 PVR_DPF(PVR_DBG_ERROR,
 628                          "BM_CreateContext: MMUInitialise failed");
 629                 goto cleanup;
 630         }
 631
 632         if (bKernelContext) {
 633                 PVR_ASSERT(psDevMemoryInfo->pBMKernelContext == NULL);
 634                 psDevMemoryInfo->pBMKernelContext = pBMContext;
 635         } else {
 636
 637                 PVR_ASSERT(psDevMemoryInfo->pBMKernelContext);
 638
 639                 if (psDevMemoryInfo->pBMKernelContext == NULL) {
 640                         PVR_DPF(PVR_DBG_ERROR, "BM_CreateContext: "
 641                                 "psDevMemoryInfo->pBMKernelContext invalid");
 642                         goto cleanup;
 643                 }
 644
 645                 PVR_ASSERT(psDevMemoryInfo->pBMKernelContext->psBMHeap);
 646
 647                 pBMContext->psBMSharedHeap =
 648                     psDevMemoryInfo->pBMKernelContext->psBMHeap;
 649
 650                 psBMHeap = pBMContext->psBMSharedHeap;
 651                 while (psBMHeap) {
 652                         switch (psBMHeap->sDevArena.DevMemHeapType) {
 653                         case DEVICE_MEMORY_HEAP_SHARED:
 654                         case DEVICE_MEMORY_HEAP_SHARED_EXPORTED:
 655                                 {
 656                                         psDeviceNode->
 657                                             pfnMMUInsertHeap(pBMContext->
 658                                                                  psMMUContext,
 659                                                              psBMHeap->
 660                                                                  pMMUHeap);
 661                                         break;
 662                                 }
 663                         }
 664                         psBMHeap = psBMHeap->psNext;
 665                 }
 666                 pBMContext->psNext = psDevMemoryInfo->pBMContext;
 667                 psDevMemoryInfo->pBMContext = pBMContext;
 668         }
 669         pBMContext->ui32RefCount++;
 670         pBMContext->hResItem = ResManRegisterRes(hResManContext,
 671                                                  RESMAN_TYPE_DEVICEMEM_CONTEXT,
 672                                                  pBMContext,
 673                                                  0, BM_DestroyContextCallBack);
 674         if (pBMContext->hResItem == NULL) {
 675                 PVR_DPF(PVR_DBG_ERROR,
 676                          "BM_CreateContext: ResManRegisterRes failed");
 677                 goto cleanup;
 678         }
 679
 680         if (pbCreated != NULL)
 681                 *pbCreated = IMG_TRUE;
 682         return (void *)pBMContext;
 683
 684 cleanup:
 685         BM_DestroyContextCallBack(pBMContext, 0);
 686
 687         return NULL;
 688 }
 689
 690 void *BM_CreateHeap(void *hBMContext,
 691                     struct DEVICE_MEMORY_HEAP_INFO *psDevMemHeapInfo)
 692 {
 693         struct BM_CONTEXT *pBMContext = (struct BM_CONTEXT *)hBMContext;
 694         struct PVRSRV_DEVICE_NODE *psDeviceNode = pBMContext->psDeviceNode;
 695         struct BM_HEAP *psBMHeap;
 696
 697         PVR_DPF(PVR_DBG_MESSAGE, "BM_CreateHeap");
 698
 699         if (pBMContext->ui32RefCount > 0) {
 700                 psBMHeap = pBMContext->psBMHeap;
 701
 702                 while (psBMHeap) {
 703                         if (psBMHeap->sDevArena.ui32HeapID ==
 704                             psDevMemHeapInfo->ui32HeapID)
 705
 706                                 return psBMHeap;
 707                         psBMHeap = psBMHeap->psNext;
 708                 }
 709         }
 710
 711         if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_HEAP),
 712                        (void **) &psBMHeap, NULL) != PVRSRV_OK) {
 713                 PVR_DPF(PVR_DBG_ERROR, "BM_CreateHeap: Alloc failed");
 714                 return NULL;
 715         }
 716
 717         OSMemSet(psBMHeap, 0, sizeof(struct BM_HEAP));
 718
 719         psBMHeap->sDevArena.ui32HeapID = psDevMemHeapInfo->ui32HeapID;
 720         psBMHeap->sDevArena.pszName = psDevMemHeapInfo->pszName;
 721         psBMHeap->sDevArena.BaseDevVAddr = psDevMemHeapInfo->sDevVAddrBase;
 722         psBMHeap->sDevArena.ui32Size = psDevMemHeapInfo->ui32HeapSize;
 723         psBMHeap->sDevArena.DevMemHeapType = psDevMemHeapInfo->DevMemHeapType;
 724         psBMHeap->sDevArena.ui32DataPageSize =
 725             psDevMemHeapInfo->ui32DataPageSize;
 726         psBMHeap->sDevArena.psDeviceMemoryHeapInfo = psDevMemHeapInfo;
 727         psBMHeap->ui32Attribs = psDevMemHeapInfo->ui32Attribs;
 728
 729         psBMHeap->pBMContext = pBMContext;
 730
 731         psBMHeap->pMMUHeap =
 732             psDeviceNode->pfnMMUCreate(pBMContext->psMMUContext,
 733                                        &psBMHeap->sDevArena,
 734                                        &psBMHeap->pVMArena);
 735         if (!psBMHeap->pMMUHeap) {
 736                 PVR_DPF(PVR_DBG_ERROR, "BM_CreateHeap: MMUCreate failed");
 737                 goto ErrorExit;
 738         }
 739
 740         psBMHeap->pImportArena = RA_Create(psDevMemHeapInfo->pszBSName,
 741                                            0, 0, NULL,
 742                                            psBMHeap->sDevArena.ui32DataPageSize,
 743                                            BM_ImportMemory,
 744                                            BM_FreeMemory, NULL, psBMHeap);
 745         if (psBMHeap->pImportArena == NULL) {
 746                 PVR_DPF(PVR_DBG_ERROR, "BM_CreateHeap: RA_Create failed");
 747                 goto ErrorExit;
 748         }
 749
 750         if (psBMHeap->ui32Attribs & PVRSRV_BACKINGSTORE_LOCALMEM_CONTIG) {
 751
 752                 psBMHeap->pLocalDevMemArena =
 753                     psDevMemHeapInfo->psLocalDevMemArena;
 754                 if (psBMHeap->pLocalDevMemArena == NULL) {
 755                         PVR_DPF(PVR_DBG_ERROR,
 756                                  "BM_CreateHeap: LocalDevMemArena null");
 757                         goto ErrorExit;
 758                 }
 759         }
 760
 761         psBMHeap->psNext = pBMContext->psBMHeap;
 762         pBMContext->psBMHeap = psBMHeap;
 763
 764         return (void *)psBMHeap;
 765
 766 ErrorExit:
 767
 768         if (psBMHeap->pMMUHeap != NULL) {
 769                 psDeviceNode->pfnMMUDelete(psBMHeap->pMMUHeap);
 770                 psDeviceNode->pfnMMUFinalise(pBMContext->psMMUContext);
 771         }
 772
 773         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_HEAP),
 774                   psBMHeap, NULL);
 775
 776         return NULL;
 777 }
 778
 779 void BM_DestroyHeap(void *hDevMemHeap)
 780 {
 781         struct BM_HEAP *psBMHeap = (struct BM_HEAP *)hDevMemHeap;
 782         struct PVRSRV_DEVICE_NODE *psDeviceNode =
 783                                         psBMHeap->pBMContext->psDeviceNode;
 784         struct BM_HEAP **ppsBMHeap;
 785
 786         PVR_DPF(PVR_DBG_MESSAGE, "BM_DestroyHeap");
 787
 788         if (psBMHeap->ui32Attribs &
 789                         (PVRSRV_BACKINGSTORE_SYSMEM_NONCONTIG |
 790                          PVRSRV_BACKINGSTORE_LOCALMEM_CONTIG)) {
 791                 if (psBMHeap->pImportArena)
 792                         RA_Delete(psBMHeap->pImportArena);
 793         } else {
 794                 PVR_DPF(PVR_DBG_ERROR,
 795                         "BM_DestroyHeap: backing store type unsupported");
 796                 return;
 797         }
 798
 799         psDeviceNode->pfnMMUDelete(psBMHeap->pMMUHeap);
 800
 801         ppsBMHeap = &psBMHeap->pBMContext->psBMHeap;
 802         while (*ppsBMHeap) {
 803                 if (*ppsBMHeap == psBMHeap) {
 804                         *ppsBMHeap = psBMHeap->psNext;
 805                         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP,
 806                                         sizeof(struct BM_HEAP), psBMHeap,
 807                                         NULL);
 808                         break;
 809                 }
 810                 ppsBMHeap = &((*ppsBMHeap)->psNext);
 811         }
 812 }
 813
 814 IMG_BOOL BM_Reinitialise(struct PVRSRV_DEVICE_NODE *psDeviceNode)
 815 {
 816         PVR_DPF(PVR_DBG_MESSAGE, "BM_Reinitialise");
 817         PVR_UNREFERENCED_PARAMETER(psDeviceNode);
 818
 819         return IMG_TRUE;
 820 }
 821
 822 IMG_BOOL BM_Alloc(void *hDevMemHeap, struct IMG_DEV_VIRTADDR *psDevVAddr,
 823                   size_t uSize, u32 *pui32Flags, u32 uDevVAddrAlignment,
 824                   void **phBuf)
 825 {
 826         struct BM_BUF *pBuf;
 827         struct BM_CONTEXT *pBMContext;
 828         struct BM_HEAP *psBMHeap;
 829         struct SYS_DATA *psSysData;
 830         u32 uFlags;
 831
 832         if (pui32Flags == NULL) {
 833                 PVR_DPF(PVR_DBG_ERROR, "BM_Alloc: invalid parameter");
 834                 PVR_DBG_BREAK;
 835                 return IMG_FALSE;
 836         }
 837
 838         uFlags = *pui32Flags;
 839
 840         PVR_DPF(PVR_DBG_MESSAGE,
 841                  "BM_Alloc (uSize=0x%x, uFlags=0x%x, uDevVAddrAlignment=0x%x)",
 842                  uSize, uFlags, uDevVAddrAlignment);
 843
 844         if (SysAcquireData(&psSysData) != PVRSRV_OK)
 845                 return IMG_FALSE;
 846
 847         psBMHeap = (struct BM_HEAP *)hDevMemHeap;
 848         pBMContext = psBMHeap->pBMContext;
 849
 850         if (uDevVAddrAlignment == 0)
 851                 uDevVAddrAlignment = 1;
 852
 853         if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_BUF),
 854                        (void **)&pBuf, NULL) != PVRSRV_OK) {
 855                 PVR_DPF(PVR_DBG_ERROR, "BM_Alloc: BM_Buf alloc FAILED");
 856                 return IMG_FALSE;
 857         }
 858         OSMemSet(pBuf, 0, sizeof(struct BM_BUF));
 859
 860         if (AllocMemory(pBMContext, psBMHeap, psDevVAddr, uSize, uFlags,
 861                         uDevVAddrAlignment, pBuf) != IMG_TRUE) {
 862                 OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_BUF), pBuf,
 863                           NULL);
 864                 PVR_DPF(PVR_DBG_ERROR, "BM_Alloc: AllocMemory FAILED");
 865                 return IMG_FALSE;
 866         }
 867
 868         PVR_DPF(PVR_DBG_MESSAGE, "BM_Alloc (uSize=0x%x, uFlags=0x%x)=%08X",
 869                  uSize, uFlags, pBuf);
 870
 871         pBuf->ui32RefCount = 1;
 872         pvr_get_ctx(pBMContext);
 873         *phBuf = (void *) pBuf;
 874         *pui32Flags = uFlags | psBMHeap->ui32Attribs;
 875
 876         return IMG_TRUE;
 877 }
 878
 879 IMG_BOOL BM_Wrap(void *hDevMemHeap, u32 ui32Size, u32 ui32Offset,
 880                  IMG_BOOL bPhysContig, struct IMG_SYS_PHYADDR *psSysAddr,
 881                  void *pvCPUVAddr, u32 *pui32Flags, void **phBuf)
 882 {
 883         struct BM_BUF *pBuf;
 884         struct BM_CONTEXT *psBMContext;
 885         struct BM_HEAP *psBMHeap;
 886         struct SYS_DATA *psSysData;
 887         struct IMG_SYS_PHYADDR sHashAddress;
 888         u32 uFlags;
 889
 890         psBMHeap = (struct BM_HEAP *)hDevMemHeap;
 891         psBMContext = psBMHeap->pBMContext;
 892
 893         uFlags = psBMHeap->ui32Attribs &
 894                  (PVRSRV_HAP_CACHETYPE_MASK | PVRSRV_HAP_MAPTYPE_MASK);
 895
 896         if (pui32Flags)
 897                 uFlags |= *pui32Flags;
 898
 899         PVR_DPF(PVR_DBG_MESSAGE, "BM_Wrap (uSize=0x%x, uOffset=0x%x, "
 900                         "bPhysContig=0x%x, pvCPUVAddr=0x%x, uFlags=0x%x)",
 901                         ui32Size, ui32Offset, bPhysContig, pvCPUVAddr, uFlags);
 902
 903         if (SysAcquireData(&psSysData) != PVRSRV_OK)
 904                 return IMG_FALSE;
 905
 906         sHashAddress = psSysAddr[0];
 907
 908         sHashAddress.uiAddr += ui32Offset;
 909
 910         pBuf = (struct BM_BUF *)HASH_Retrieve(psBMContext->pBufferHash,
 911                                      (u32) sHashAddress.uiAddr);
 912
 913         if (pBuf) {
 914                 u32 ui32MappingSize =
 915                     HOST_PAGEALIGN(ui32Size + ui32Offset);
 916
 917                 if (pBuf->pMapping->uSize == ui32MappingSize &&
 918                     (pBuf->pMapping->eCpuMemoryOrigin == hm_wrapped ||
 919                       pBuf->pMapping->eCpuMemoryOrigin ==
 920                         hm_wrapped_virtaddr)) {
 921                         PVR_DPF(PVR_DBG_MESSAGE, "BM_Wrap "
 922                                 "(Matched previous Wrap! uSize=0x%x, "
 923                                 "uOffset=0x%x, SysAddr=%08X)",
 924                                  ui32Size, ui32Offset, sHashAddress.uiAddr);
 925
 926                         pBuf->ui32RefCount++;
 927                         *phBuf = (void *)pBuf;
 928                         if (pui32Flags)
 929                                 *pui32Flags = uFlags;
 930
 931                         return IMG_TRUE;
 932                 }
 933         }
 934
 935         if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_BUF),
 936                        (void **)&pBuf, NULL) != PVRSRV_OK) {
 937                 PVR_DPF(PVR_DBG_ERROR, "BM_Wrap: BM_Buf alloc FAILED");
 938                 return IMG_FALSE;
 939         }
 940         OSMemSet(pBuf, 0, sizeof(struct BM_BUF));
 941
 942         if (WrapMemory(psBMHeap, ui32Size, ui32Offset, bPhysContig, psSysAddr,
 943                         pvCPUVAddr, uFlags, pBuf) != IMG_TRUE) {
 944                 PVR_DPF(PVR_DBG_ERROR, "BM_Wrap: WrapMemory FAILED");
 945                 OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_BUF), pBuf,
 946                           NULL);
 947                 return IMG_FALSE;
 948         }
 949
 950         if (pBuf->pMapping->eCpuMemoryOrigin == hm_wrapped ||
 951             pBuf->pMapping->eCpuMemoryOrigin == hm_wrapped_virtaddr) {
 952
 953                 PVR_ASSERT(SysSysPAddrToCpuPAddr(sHashAddress).uiAddr ==
 954                            pBuf->CpuPAddr.uiAddr);
 955
 956                 if (!HASH_Insert(psBMContext->pBufferHash,
 957                                  (u32)sHashAddress.uiAddr, (u32) pBuf)) {
 958                         FreeBuf(pBuf, uFlags);
 959                         PVR_DPF(PVR_DBG_ERROR, "BM_Wrap: HASH_Insert FAILED");
 960                         return IMG_FALSE;
 961                 }
 962         }
 963
 964         PVR_DPF(PVR_DBG_MESSAGE,
 965                  "BM_Wrap (uSize=0x%x, uFlags=0x%x)=%08X(devVAddr=%08X)",
 966                  ui32Size, uFlags, pBuf, pBuf->DevVAddr.uiAddr);
 967
 968         pBuf->ui32RefCount = 1;
 969         pvr_get_ctx(psBMContext);
 970         *phBuf = (void *) pBuf;
 971         if (pui32Flags)
 972                 *pui32Flags = (uFlags & ~PVRSRV_HAP_MAPTYPE_MASK) |
 973                               PVRSRV_HAP_MULTI_PROCESS;
 974
 975         return IMG_TRUE;
 976 }
 977
 978 void BM_Free(void *hBuf, u32 ui32Flags)
 979 {
 980         struct BM_BUF *pBuf = (struct BM_BUF *)hBuf;
 981         struct SYS_DATA *psSysData;
 982         struct IMG_SYS_PHYADDR sHashAddr;
 983
 984         PVR_DPF(PVR_DBG_MESSAGE, "BM_Free (h=%08X)", hBuf);
 985         PVR_ASSERT(pBuf != NULL);
 986
 987         if (pBuf == NULL) {
 988                 PVR_DPF(PVR_DBG_ERROR, "BM_Free: invalid parameter");
 989                 return;
 990         }
 991
 992         if (SysAcquireData(&psSysData) != PVRSRV_OK)
 993                 return;
 994
 995         pBuf->ui32RefCount--;
 996
 997         if (pBuf->ui32RefCount == 0) {
 998                 struct BM_MAPPING *map = pBuf->pMapping;
 999                 struct BM_CONTEXT *ctx = map->pBMHeap->pBMContext;
1000
1001                 if (map->eCpuMemoryOrigin == hm_wrapped ||
1002                     map->eCpuMemoryOrigin == hm_wrapped_virtaddr) {
1003                         sHashAddr = SysCpuPAddrToSysPAddr(pBuf->CpuPAddr);
1004
1005                         HASH_Remove(ctx->pBufferHash, (u32)sHashAddr.uiAddr);
1006                 }
1007                 FreeBuf(pBuf, ui32Flags);
1008                 pvr_put_ctx(ctx);
1009         }
1010 }
1011
1012 void *BM_HandleToCpuVaddr(void *hBuf)
1013 {
1014         struct BM_BUF *pBuf = (struct BM_BUF *)hBuf;
1015
1016         PVR_ASSERT(pBuf != NULL);
1017         if (pBuf == NULL) {
1018                 PVR_DPF(PVR_DBG_ERROR,
1019                          "BM_HandleToCpuVaddr: invalid parameter");
1020                 return NULL;
1021         }
1022
1023         PVR_DPF(PVR_DBG_MESSAGE,
1024                  "BM_HandleToCpuVaddr(h=%08X)=%08X", hBuf, pBuf->CpuVAddr);
1025         return pBuf->CpuVAddr;
1026 }
1027
1028 struct IMG_DEV_VIRTADDR BM_HandleToDevVaddr(void *hBuf)
1029 {
1030         struct BM_BUF *pBuf = (struct BM_BUF *)hBuf;
1031
1032         PVR_ASSERT(pBuf != NULL);
1033         if (pBuf == NULL) {
1034                 struct IMG_DEV_VIRTADDR DevVAddr = { 0 };
1035                 PVR_DPF(PVR_DBG_ERROR,
1036                          "BM_HandleToDevVaddr: invalid parameter");
1037                 return DevVAddr;
1038         }
1039
1040         PVR_DPF(PVR_DBG_MESSAGE, "BM_HandleToDevVaddr(h=%08X)=%08X", hBuf,
1041                  pBuf->DevVAddr);
1042         return pBuf->DevVAddr;
1043 }
1044
1045 struct IMG_SYS_PHYADDR BM_HandleToSysPaddr(void *hBuf)
1046 {
1047         struct BM_BUF *pBuf = (struct BM_BUF *)hBuf;
1048
1049         PVR_ASSERT(pBuf != NULL);
1050
1051         if (pBuf == NULL) {
1052                 struct IMG_SYS_PHYADDR PhysAddr = { 0 };
1053                 PVR_DPF(PVR_DBG_ERROR,
1054                          "BM_HandleToSysPaddr: invalid parameter");
1055                 return PhysAddr;
1056         }
1057
1058         PVR_DPF(PVR_DBG_MESSAGE, "BM_HandleToSysPaddr(h=%08X)=%08X", hBuf,
1059                  pBuf->CpuPAddr.uiAddr);
1060         return SysCpuPAddrToSysPAddr(pBuf->CpuPAddr);
1061 }
1062
1063 void *BM_HandleToOSMemHandle(void *hBuf)
1064 {
1065         struct BM_BUF *pBuf = (struct BM_BUF *)hBuf;
1066
1067         PVR_ASSERT(pBuf != NULL);
1068
1069         if (pBuf == NULL) {
1070                 PVR_DPF(PVR_DBG_ERROR,
1071                          "BM_HandleToOSMemHandle: invalid parameter");
1072                 return NULL;
1073         }
1074
1075         PVR_DPF(PVR_DBG_MESSAGE,
1076                  "BM_HandleToOSMemHandle(h=%08X)=%08X",
1077                  hBuf, pBuf->hOSMemHandle);
1078         return pBuf->hOSMemHandle;
1079 }
1080
1081 static IMG_BOOL DevMemoryAlloc(struct BM_CONTEXT *pBMContext,
1082                struct BM_MAPPING *pMapping, u32 uFlags, u32 dev_vaddr_alignment,
1083                struct IMG_DEV_VIRTADDR *pDevVAddr)
1084 {
1085         struct PVRSRV_DEVICE_NODE *psDeviceNode = pBMContext->psDeviceNode;
1086
1087         if (uFlags & PVRSRV_MEM_INTERLEAVED)
1088                 pMapping->uSize *= 2;
1089
1090         if (!psDeviceNode->pfnMMUAlloc(pMapping->pBMHeap->pMMUHeap,
1091                                        pMapping->uSize, 0, dev_vaddr_alignment,
1092                                        &(pMapping->DevVAddr))) {
1093                 PVR_DPF(PVR_DBG_ERROR, "DevMemoryAlloc ERROR MMU_Alloc");
1094                 return IMG_FALSE;
1095         }
1096
1097 #ifdef PDUMP
1098         {
1099                 u32 ui32PDumpSize = pMapping->uSize;
1100
1101                 if (uFlags & PVRSRV_MEM_DUMMY)
1102                         ui32PDumpSize =
1103                                 pMapping->pBMHeap->sDevArena.ui32DataPageSize;
1104
1105                 PDUMPMALLOCPAGES(pMapping->DevVAddr.uiAddr,
1106                                  pMapping->hOSMemHandle, ui32PDumpSize,
1107                                  (void *)pMapping);
1108         }
1109 #endif
1110
1111         switch (pMapping->eCpuMemoryOrigin) {
1112         case hm_wrapped:
1113         case hm_wrapped_virtaddr:
1114         case hm_contiguous:
1115                 {
1116                         psDeviceNode->pfnMMUMapPages(pMapping->pBMHeap->
1117                                                            pMMUHeap,
1118                                                      pMapping->DevVAddr,
1119                                                      SysCpuPAddrToSysPAddr
1120                                                            (pMapping->CpuPAddr),
1121                                                      pMapping->uSize, uFlags,
1122                                                      (void *)pMapping);
1123
1124                         *pDevVAddr = pMapping->DevVAddr;
1125                         break;
1126                 }
1127         case hm_env:
1128                 {
1129                         psDeviceNode->pfnMMUMapShadow(pMapping->pBMHeap->
1130                                                             pMMUHeap,
1131                                                       pMapping->DevVAddr,
1132                                                       pMapping->uSize,
1133                                                       pMapping->CpuVAddr,
1134                                                       pMapping->hOSMemHandle,
1135                                                       pDevVAddr, uFlags,
1136                                                       (void *)pMapping);
1137                         break;
1138                 }
1139         case hm_wrapped_scatter:
1140         case hm_wrapped_scatter_virtaddr:
1141                 {
1142                         psDeviceNode->pfnMMUMapScatter(pMapping->pBMHeap->
1143                                                              pMMUHeap,
1144                                                        pMapping->DevVAddr,
1145                                                        pMapping->psSysAddr,
1146                                                        pMapping->uSize, uFlags,
1147                                                        (void *)pMapping);
1148
1149                         *pDevVAddr = pMapping->DevVAddr;
1150                         break;
1151                 }
1152         default:
1153                 PVR_DPF(PVR_DBG_ERROR,
1154                          "Illegal value %d for pMapping->eCpuMemoryOrigin",
1155                          pMapping->eCpuMemoryOrigin);
1156                 return IMG_FALSE;
1157         }
1158
1159
1160         return IMG_TRUE;
1161 }
1162
1163 static void DevMemoryFree(struct BM_MAPPING *pMapping)
1164 {
1165         struct PVRSRV_DEVICE_NODE *psDeviceNode;
1166 #ifdef PDUMP
1167         u32 ui32PSize;
1168
1169         if (pMapping->ui32Flags & PVRSRV_MEM_DUMMY)
1170                 ui32PSize = pMapping->pBMHeap->sDevArena.ui32DataPageSize;
1171         else
1172                 ui32PSize = pMapping->uSize;
1173
1174         PDUMPFREEPAGES(pMapping->pBMHeap, pMapping->DevVAddr, ui32PSize,
1175                        (void *)pMapping, (IMG_BOOL)(pMapping->
1176                                    ui32Flags & PVRSRV_MEM_INTERLEAVED));
1177 #endif
1178
1179         psDeviceNode = pMapping->pBMHeap->pBMContext->psDeviceNode;
1180
1181         psDeviceNode->pfnMMUFree(pMapping->pBMHeap->pMMUHeap,
1182                                  pMapping->DevVAddr, pMapping->uSize);
1183 }
1184
1185 static IMG_BOOL BM_ImportMemory(void *pH, size_t uRequestSize,
1186                 size_t *pActualSize, struct BM_MAPPING **ppsMapping,
1187                 u32 uFlags, u32 *pBase)
1188 {
1189         struct BM_MAPPING *pMapping;
1190         struct BM_HEAP *pBMHeap = pH;
1191         struct BM_CONTEXT *pBMContext = pBMHeap->pBMContext;
1192         IMG_BOOL bResult;
1193         size_t uSize;
1194         size_t uPSize;
1195         u32 uDevVAddrAlignment = 0;
1196
1197         PVR_DPF(PVR_DBG_MESSAGE,
1198                 "BM_ImportMemory (pBMContext=%08X, uRequestSize=0x%x, "
1199                 "uFlags=0x%x, uAlign=0x%x)",
1200                  pBMContext, uRequestSize, uFlags, uDevVAddrAlignment);
1201
1202         PVR_ASSERT(ppsMapping != NULL);
1203         PVR_ASSERT(pBMContext != NULL);
1204
1205         if (ppsMapping == NULL) {
1206                 PVR_DPF(PVR_DBG_ERROR, "BM_ImportMemory: invalid parameter");
1207                 goto fail_exit;
1208         }
1209
1210         uSize = HOST_PAGEALIGN(uRequestSize);
1211         PVR_ASSERT(uSize >= uRequestSize);
1212
1213         if (OSAllocMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_MAPPING),
1214                        (void **)&pMapping, NULL) != PVRSRV_OK) {
1215                 PVR_DPF(PVR_DBG_ERROR,
1216                          "BM_ImportMemory: failed struct BM_MAPPING alloc");
1217                 goto fail_exit;
1218         }
1219
1220         pMapping->hOSMemHandle = NULL;
1221         pMapping->CpuVAddr = NULL;
1222         pMapping->DevVAddr.uiAddr = 0;
1223         pMapping->CpuPAddr.uiAddr = 0;
1224         pMapping->uSize = uSize;
1225         pMapping->pBMHeap = pBMHeap;
1226         pMapping->ui32Flags = uFlags;
1227
1228         if (pActualSize)
1229                 *pActualSize = uSize;
1230
1231         if (pMapping->ui32Flags & PVRSRV_MEM_DUMMY)
1232                 uPSize = pBMHeap->sDevArena.ui32DataPageSize;
1233         else
1234                 uPSize = pMapping->uSize;
1235
1236         if (pBMHeap->ui32Attribs & PVRSRV_BACKINGSTORE_SYSMEM_NONCONTIG) {
1237                 if (OSAllocPages(pBMHeap->ui32Attribs, uPSize,
1238                                  pBMHeap->sDevArena.ui32DataPageSize,
1239                                  (void **)&pMapping->CpuVAddr,
1240                                  &pMapping->hOSMemHandle) != PVRSRV_OK) {
1241                         PVR_DPF(PVR_DBG_ERROR,
1242                                  "BM_ImportMemory: OSAllocPages(0x%x) failed",
1243                                  uPSize);
1244                         goto fail_mapping_alloc;
1245                 }
1246
1247                 pMapping->eCpuMemoryOrigin = hm_env;
1248         } else if (pBMHeap->ui32Attribs & PVRSRV_BACKINGSTORE_LOCALMEM_CONTIG) {
1249                 struct IMG_SYS_PHYADDR sSysPAddr;
1250
1251                 PVR_ASSERT(pBMHeap->pLocalDevMemArena != NULL);
1252
1253                 if (!RA_Alloc(pBMHeap->pLocalDevMemArena, uPSize, NULL, 0,
1254                               pBMHeap->sDevArena.ui32DataPageSize,
1255                               (u32 *)&sSysPAddr.uiAddr)) {
1256                         PVR_DPF(PVR_DBG_ERROR,
1257                                  "BM_ImportMemory: RA_Alloc(0x%x) FAILED",
1258                                  uPSize);
1259                         goto fail_mapping_alloc;
1260                 }
1261
1262                 pMapping->CpuPAddr = SysSysPAddrToCpuPAddr(sSysPAddr);
1263                 if (OSReservePhys(pMapping->CpuPAddr, uPSize,
1264                                   pBMHeap->ui32Attribs, &pMapping->CpuVAddr,
1265                                   &pMapping->hOSMemHandle) != PVRSRV_OK) {
1266                         PVR_DPF(PVR_DBG_ERROR,
1267                                  "BM_ImportMemory: OSReservePhys failed");
1268                         goto fail_dev_mem_alloc;
1269                 }
1270
1271                 pMapping->eCpuMemoryOrigin = hm_contiguous;
1272         } else {
1273                 PVR_DPF(PVR_DBG_ERROR,
1274                          "BM_ImportMemory: Invalid backing store type");
1275                 goto fail_mapping_alloc;
1276         }
1277
1278         bResult = DevMemoryAlloc(pBMContext, pMapping, uFlags,
1279                                  uDevVAddrAlignment, &pMapping->DevVAddr);
1280         if (!bResult) {
1281                 PVR_DPF(PVR_DBG_ERROR,
1282                          "BM_ImportMemory: DevMemoryAlloc(0x%x) failed",
1283                          pMapping->uSize);
1284                 goto fail_dev_mem_alloc;
1285         }
1286
1287         PVR_ASSERT(uDevVAddrAlignment > 1 ?
1288                    (pMapping->DevVAddr.uiAddr % uDevVAddrAlignment) == 0 : 1);
1289
1290         *pBase = pMapping->DevVAddr.uiAddr;
1291         *ppsMapping = pMapping;
1292
1293         PVR_DPF(PVR_DBG_MESSAGE, "BM_ImportMemory: IMG_TRUE");
1294         return IMG_TRUE;
1295
1296 fail_dev_mem_alloc:
1297         if (pMapping->CpuVAddr || pMapping->hOSMemHandle) {
1298                 if (pMapping->ui32Flags & PVRSRV_MEM_INTERLEAVED)
1299                         pMapping->uSize /= 2;
1300
1301                 if (pMapping->ui32Flags & PVRSRV_MEM_DUMMY)
1302                         uPSize = pBMHeap->sDevArena.ui32DataPageSize;
1303                 else
1304                         uPSize = pMapping->uSize;
1305
1306                 if (pBMHeap->ui32Attribs &
1307                         PVRSRV_BACKINGSTORE_SYSMEM_NONCONTIG) {
1308                         OSFreePages(pBMHeap->ui32Attribs, uPSize,
1309                                     (void *)pMapping->CpuVAddr,
1310                                     pMapping->hOSMemHandle);
1311                 } else {
1312                         struct IMG_SYS_PHYADDR sSysPAddr;
1313
1314                         if (pMapping->CpuVAddr)
1315                                 OSUnReservePhys(pMapping->CpuVAddr, uPSize,
1316                                                 pBMHeap->ui32Attribs,
1317                                                 pMapping->hOSMemHandle);
1318                         sSysPAddr = SysCpuPAddrToSysPAddr(pMapping->CpuPAddr);
1319                         RA_Free(pBMHeap->pLocalDevMemArena, sSysPAddr.uiAddr,
1320                                 IMG_FALSE);
1321                 }
1322         }
1323 fail_mapping_alloc:
1324         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_MAPPING), pMapping,
1325                   NULL);
1326 fail_exit:
1327         return IMG_FALSE;
1328 }
1329
1330 static void BM_FreeMemory(void *h, u32 _base, struct BM_MAPPING *psMapping)
1331 {
1332         struct BM_HEAP *pBMHeap = h;
1333         size_t uPSize;
1334
1335         PVR_UNREFERENCED_PARAMETER(_base);
1336
1337         PVR_DPF(PVR_DBG_MESSAGE,
1338                  "BM_FreeMemory (h=%08X, base=0x%x, psMapping=0x%x)", h, _base,
1339                  psMapping);
1340
1341         PVR_ASSERT(psMapping != NULL);
1342
1343         if (psMapping == NULL) {
1344                 PVR_DPF(PVR_DBG_ERROR, "BM_FreeMemory: invalid parameter");
1345                 return;
1346         }
1347
1348         DevMemoryFree(psMapping);
1349
1350         if ((psMapping->ui32Flags & PVRSRV_MEM_INTERLEAVED) != 0)
1351                 psMapping->uSize /= 2;
1352
1353         if (psMapping->ui32Flags & PVRSRV_MEM_DUMMY)
1354                 uPSize = psMapping->pBMHeap->sDevArena.ui32DataPageSize;
1355         else
1356                 uPSize = psMapping->uSize;
1357
1358         if (pBMHeap->ui32Attribs & PVRSRV_BACKINGSTORE_SYSMEM_NONCONTIG) {
1359                 OSFreePages(pBMHeap->ui32Attribs, uPSize,
1360                             (void *)psMapping->CpuVAddr,
1361                             psMapping->hOSMemHandle);
1362         } else if (pBMHeap->ui32Attribs & PVRSRV_BACKINGSTORE_LOCALMEM_CONTIG) {
1363                 struct IMG_SYS_PHYADDR sSysPAddr;
1364
1365                 OSUnReservePhys(psMapping->CpuVAddr, uPSize,
1366                                 pBMHeap->ui32Attribs, psMapping->hOSMemHandle);
1367
1368                 sSysPAddr = SysCpuPAddrToSysPAddr(psMapping->CpuPAddr);
1369
1370                 RA_Free(pBMHeap->pLocalDevMemArena, sSysPAddr.uiAddr,
1371                         IMG_FALSE);
1372         } else {
1373                 PVR_DPF(PVR_DBG_ERROR,
1374                          "BM_FreeMemory: Invalid backing store type");
1375         }
1376
1377         OSFreeMem(PVRSRV_OS_PAGEABLE_HEAP, sizeof(struct BM_MAPPING), psMapping,
1378                   NULL);
1379
1380         PVR_DPF(PVR_DBG_MESSAGE,
1381                  "..BM_FreeMemory (h=%08X, base=0x%x, psMapping=0x%x)",
1382                  h, _base, psMapping);
1383 }
1384
1385 enum PVRSRV_ERROR BM_GetPhysPageAddr(struct PVRSRV_KERNEL_MEM_INFO *psMemInfo,
1386                                 struct IMG_DEV_VIRTADDR sDevVPageAddr,
1387                                 struct IMG_DEV_PHYADDR *psDevPAddr)
1388 {
1389         struct PVRSRV_DEVICE_NODE *psDeviceNode;
1390
1391         PVR_DPF(PVR_DBG_MESSAGE, "BM_GetPhysPageAddr");
1392
1393         if (!psMemInfo || !psDevPAddr) {
1394                 PVR_DPF(PVR_DBG_ERROR, "BM_GetPhysPageAddr: Invalid params");
1395                 return PVRSRV_ERROR_INVALID_PARAMS;
1396         }
1397
1398         psDeviceNode =
1399             ((struct BM_BUF *)psMemInfo->sMemBlk.hBuffer)->pMapping->pBMHeap->
1400             pBMContext->psDeviceNode;
1401
1402         *psDevPAddr = psDeviceNode->pfnMMUGetPhysPageAddr(((struct BM_BUF *)
1403                                 psMemInfo->sMemBlk.hBuffer)->
1404                                 pMapping->pBMHeap->pMMUHeap, sDevVPageAddr);
1405
1406         return PVRSRV_OK;
1407 }
1408
1409 enum PVRSRV_ERROR BM_GetHeapInfo(void *hDevMemHeap,
1410                             struct PVRSRV_HEAP_INFO *psHeapInfo)
1411 {
1412         struct BM_HEAP *psBMHeap = (struct BM_HEAP *)hDevMemHeap;
1413
1414         PVR_DPF(PVR_DBG_VERBOSE, "BM_GetHeapInfo");
1415
1416         psHeapInfo->hDevMemHeap = hDevMemHeap;
1417         psHeapInfo->sDevVAddrBase = psBMHeap->sDevArena.BaseDevVAddr;
1418         psHeapInfo->ui32HeapByteSize = psBMHeap->sDevArena.ui32Size;
1419         psHeapInfo->ui32Attribs = psBMHeap->ui32Attribs;
1420
1421         return PVRSRV_OK;
1422 }
1423
1424 struct MMU_CONTEXT *BM_GetMMUContext(void *hDevMemHeap)
1425 {
1426         struct BM_HEAP *pBMHeap = (struct BM_HEAP *)hDevMemHeap;
1427
1428         PVR_DPF(PVR_DBG_VERBOSE, "BM_GetMMUContext");
1429
1430         return pBMHeap->pBMContext->psMMUContext;
1431 }
1432
1433 struct MMU_CONTEXT *BM_GetMMUContextFromMemContext(void *hDevMemContext)
1434 {
1435         struct BM_CONTEXT *pBMContext = (struct BM_CONTEXT *)hDevMemContext;
1436
1437         PVR_DPF(PVR_DBG_VERBOSE, "BM_GetMMUContextFromMemContext");
1438
1439         return pBMContext->psMMUContext;
1440 }
1441
1442 void *BM_GetMMUHeap(void *hDevMemHeap)
1443 {
1444         PVR_DPF(PVR_DBG_VERBOSE, "BM_GetMMUHeap");
1445
1446         return (void *)((struct BM_HEAP *)hDevMemHeap)->pMMUHeap;
1447 }
1448
1449 struct PVRSRV_DEVICE_NODE *BM_GetDeviceNode(void *hDevMemContext)
1450 {
1451         PVR_DPF(PVR_DBG_VERBOSE, "BM_GetDeviceNode");
1452
1453         return ((struct BM_CONTEXT *)hDevMemContext)->psDeviceNode;
1454 }
1455
1456 void *BM_GetMappingHandle(struct PVRSRV_KERNEL_MEM_INFO *psMemInfo)
1457 {
1458         PVR_DPF(PVR_DBG_VERBOSE, "BM_GetMappingHandle");
1459
1460         return ((struct BM_BUF *)
1461                         psMemInfo->sMemBlk.hBuffer)->pMapping->hOSMemHandle;
1462 }
1463
1464 struct BM_CONTEXT *bm_find_context(struct BM_CONTEXT *head_context,
1465                                     u32 page_dir)
1466 {
1467         struct BM_CONTEXT *context = head_context;
1468
1469         /* Walk all the contexts until we find the right one */
1470         while (context) {
1471                 if (mmu_get_page_dir(context->psMMUContext) == page_dir)
1472                                 break;
1473                 context = context->psNext;
1474         }
1475         return context;
1476 }