drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #include "ttm/ttm_bo_driver.h"
  32 #include "ttm/ttm_placement.h"
  33 #include <linux/io.h>
  34 #include <linux/highmem.h>
  35 #include <linux/wait.h>
  36 #include <linux/slab.h>
  37 #include <linux/vmalloc.h>
  38 #include <linux/module.h>
  39
  40 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  41 {
  42         ttm_bo_mem_put(bo, &bo->mem);
  43 }
  44
  45 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  46                     bool evict, bool no_wait_reserve,
  47                     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
  48 {
  49         struct ttm_tt *ttm = bo->ttm;
  50         struct ttm_mem_reg *old_mem = &bo->mem;
  51         int ret;
  52
  53         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  54                 ttm_tt_unbind(ttm);
  55                 ttm_bo_free_old_node(bo);
  56                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  57                                 TTM_PL_MASK_MEM);
  58                 old_mem->mem_type = TTM_PL_SYSTEM;
  59         }
  60
  61         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  62         if (unlikely(ret != 0))
  63                 return ret;
  64
  65         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  66                 ret = ttm_tt_bind(ttm, new_mem);
  67                 if (unlikely(ret != 0))
  68                         return ret;
  69         }
  70
  71         *old_mem = *new_mem;
  72         new_mem->mm_node = NULL;
  73
  74         return 0;
  75 }
  76 EXPORT_SYMBOL(ttm_bo_move_ttm);
  77
  78 int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  79 {
  80         int ret;
  81
  82         if (!mem->bus.io_reserved) {
  83                 mem->bus.io_reserved = true;
  84                 ret = bdev->driver->io_mem_reserve(bdev, mem);
  85                 if (unlikely(ret != 0))
  86                         return ret;
  87         }
  88         return 0;
  89 }
  90
  91 void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  92 {
  93         if (bdev->driver->io_mem_reserve) {
  94                 if (mem->bus.io_reserved) {
  95                         mem->bus.io_reserved = false;
  96                         bdev->driver->io_mem_free(bdev, mem);
  97                 }
  98         }
  99 }
 100
 101 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 102                         void **virtual)
 103 {
 104         int ret;
 105         void *addr;
 106
 107         *virtual = NULL;
 108         ret = ttm_mem_io_reserve(bdev, mem);
 109         if (ret || !mem->bus.is_iomem)
 110                 return ret;
 111
 112         if (mem->bus.addr) {
 113                 addr = mem->bus.addr;
 114         } else {
 115                 if (mem->placement & TTM_PL_FLAG_WC)
 116                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 117                 else
 118                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 119                 if (!addr) {
 120                         ttm_mem_io_free(bdev, mem);
 121                         return -ENOMEM;
 122                 }
 123         }
 124         *virtual = addr;
 125         return 0;
 126 }
 127
 128 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 129                          void *virtual)
 130 {
 131         struct ttm_mem_type_manager *man;
 132
 133         man = &bdev->man[mem->mem_type];
 134
 135         if (virtual && mem->bus.addr == NULL)
 136                 iounmap(virtual);
 137         ttm_mem_io_free(bdev, mem);
 138 }
 139
 140 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 141 {
 142         uint32_t *dstP =
 143             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 144         uint32_t *srcP =
 145             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 146
 147         int i;
 148         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 149                 iowrite32(ioread32(srcP++), dstP++);
 150         return 0;
 151 }
 152
 153 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 154                                 unsigned long page,
 155                                 pgprot_t prot)
 156 {
 157         struct page *d = ttm_tt_get_page(ttm, page);
 158         void *dst;
 159
 160         if (!d)
 161                 return -ENOMEM;
 162
 163         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 164
 165 #ifdef CONFIG_X86
 166         dst = kmap_atomic_prot(d, prot);
 167 #else
 168         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 169                 dst = vmap(&d, 1, 0, prot);
 170         else
 171                 dst = kmap(d);
 172 #endif
 173         if (!dst)
 174                 return -ENOMEM;
 175
 176         memcpy_fromio(dst, src, PAGE_SIZE);
 177
 178 #ifdef CONFIG_X86
 179         kunmap_atomic(dst);
 180 #else
 181         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 182                 vunmap(dst);
 183         else
 184                 kunmap(d);
 185 #endif
 186
 187         return 0;
 188 }
 189
 190 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 191                                 unsigned long page,
 192                                 pgprot_t prot)
 193 {
 194         struct page *s = ttm_tt_get_page(ttm, page);
 195         void *src;
 196
 197         if (!s)
 198                 return -ENOMEM;
 199
 200         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 201 #ifdef CONFIG_X86
 202         src = kmap_atomic_prot(s, prot);
 203 #else
 204         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 205                 src = vmap(&s, 1, 0, prot);
 206         else
 207                 src = kmap(s);
 208 #endif
 209         if (!src)
 210                 return -ENOMEM;
 211
 212         memcpy_toio(dst, src, PAGE_SIZE);
 213
 214 #ifdef CONFIG_X86
 215         kunmap_atomic(src);
 216 #else
 217         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 218                 vunmap(src);
 219         else
 220                 kunmap(s);
 221 #endif
 222
 223         return 0;
 224 }
 225
 226 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 227                        bool evict, bool no_wait_reserve, bool no_wait_gpu,
 228                        struct ttm_mem_reg *new_mem)
 229 {
 230         struct ttm_bo_device *bdev = bo->bdev;
 231         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 232         struct ttm_tt *ttm = bo->ttm;
 233         struct ttm_mem_reg *old_mem = &bo->mem;
 234         struct ttm_mem_reg old_copy = *old_mem;
 235         void *old_iomap;
 236         void *new_iomap;
 237         int ret;
 238         unsigned long i;
 239         unsigned long page;
 240         unsigned long add = 0;
 241         int dir;
 242
 243         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 244         if (ret)
 245                 return ret;
 246         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 247         if (ret)
 248                 goto out;
 249
 250         if (old_iomap == NULL && new_iomap == NULL)
 251                 goto out2;
 252         if (old_iomap == NULL && ttm == NULL)
 253                 goto out2;
 254
 255         add = 0;
 256         dir = 1;
 257
 258         if ((old_mem->mem_type == new_mem->mem_type) &&
 259             (new_mem->start < old_mem->start + old_mem->size)) {
 260                 dir = -1;
 261                 add = new_mem->num_pages - 1;
 262         }
 263
 264         for (i = 0; i < new_mem->num_pages; ++i) {
 265                 page = i * dir + add;
 266                 if (old_iomap == NULL) {
 267                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 268                                                     PAGE_KERNEL);
 269                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 270                                                    prot);
 271                 } else if (new_iomap == NULL) {
 272                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 273                                                     PAGE_KERNEL);
 274                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 275                                                    prot);
 276                 } else
 277                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 278                 if (ret)
 279                         goto out1;
 280         }
 281         mb();
 282 out2:
 283         ttm_bo_free_old_node(bo);
 284
 285         *old_mem = *new_mem;
 286         new_mem->mm_node = NULL;
 287
 288         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
 289                 ttm_tt_unbind(ttm);
 290                 ttm_tt_destroy(ttm);
 291                 bo->ttm = NULL;
 292         }
 293
 294 out1:
 295         ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
 296 out:
 297         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 298         return ret;
 299 }
 300 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 301
 302 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 303 {
 304         kfree(bo);
 305 }
 306
 307 /**
 308  * ttm_buffer_object_transfer
 309  *
 310  * @bo: A pointer to a struct ttm_buffer_object.
 311  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 312  * holding the data of @bo with the old placement.
 313  *
 314  * This is a utility function that may be called after an accelerated move
 315  * has been scheduled. A new buffer object is created as a placeholder for
 316  * the old data while it's being copied. When that buffer object is idle,
 317  * it can be destroyed, releasing the space of the old placement.
 318  * Returns:
 319  * !0: Failure.
 320  */
 321
 322 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 323                                       struct ttm_buffer_object **new_obj)
 324 {
 325         struct ttm_buffer_object *fbo;
 326         struct ttm_bo_device *bdev = bo->bdev;
 327         struct ttm_bo_driver *driver = bdev->driver;
 328
 329         fbo = kzalloc(sizeof(*fbo), GFP_KERNEL);
 330         if (!fbo)
 331                 return -ENOMEM;
 332
 333         *fbo = *bo;
 334
 335         /**
 336          * Fix up members that we shouldn't copy directly:
 337          * TODO: Explicit member copy would probably be better here.
 338          */
 339
 340         spin_lock_init(&fbo->lock);
 341         init_waitqueue_head(&fbo->event_queue);
 342         INIT_LIST_HEAD(&fbo->ddestroy);
 343         INIT_LIST_HEAD(&fbo->lru);
 344         INIT_LIST_HEAD(&fbo->swap);
 345         fbo->vm_node = NULL;
 346         atomic_set(&fbo->cpu_writers, 0);
 347
 348         fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
 349         kref_init(&fbo->list_kref);
 350         kref_init(&fbo->kref);
 351         fbo->destroy = &ttm_transfered_destroy;
 352
 353         *new_obj = fbo;
 354         return 0;
 355 }
 356
 357 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 358 {
 359 #if defined(__i386__) || defined(__x86_64__)
 360         if (caching_flags & TTM_PL_FLAG_WC)
 361                 tmp = pgprot_writecombine(tmp);
 362         else if (boot_cpu_data.x86 > 3)
 363                 tmp = pgprot_noncached(tmp);
 364
 365 #elif defined(__powerpc__)
 366         if (!(caching_flags & TTM_PL_FLAG_CACHED)) {
 367                 pgprot_val(tmp) |= _PAGE_NO_CACHE;
 368                 if (caching_flags & TTM_PL_FLAG_UNCACHED)
 369                         pgprot_val(tmp) |= _PAGE_GUARDED;
 370         }
 371 #endif
 372 #if defined(__ia64__)
 373         if (caching_flags & TTM_PL_FLAG_WC)
 374                 tmp = pgprot_writecombine(tmp);
 375         else
 376                 tmp = pgprot_noncached(tmp);
 377 #endif
 378 #if defined(__sparc__)
 379         if (!(caching_flags & TTM_PL_FLAG_CACHED))
 380                 tmp = pgprot_noncached(tmp);
 381 #endif
 382         return tmp;
 383 }
 384 EXPORT_SYMBOL(ttm_io_prot);
 385
 386 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 387                           unsigned long offset,
 388                           unsigned long size,
 389                           struct ttm_bo_kmap_obj *map)
 390 {
 391         struct ttm_mem_reg *mem = &bo->mem;
 392
 393         if (bo->mem.bus.addr) {
 394                 map->bo_kmap_type = ttm_bo_map_premapped;
 395                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 396         } else {
 397                 map->bo_kmap_type = ttm_bo_map_iomap;
 398                 if (mem->placement & TTM_PL_FLAG_WC)
 399                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 400                                                   size);
 401                 else
 402                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 403                                                        size);
 404         }
 405         return (!map->virtual) ? -ENOMEM : 0;
 406 }
 407
 408 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 409                            unsigned long start_page,
 410                            unsigned long num_pages,
 411                            struct ttm_bo_kmap_obj *map)
 412 {
 413         struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
 414         struct ttm_tt *ttm = bo->ttm;
 415         struct page *d;
 416         int i;
 417
 418         BUG_ON(!ttm);
 419         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 420                 /*
 421                  * We're mapping a single page, and the desired
 422                  * page protection is consistent with the bo.
 423                  */
 424
 425                 map->bo_kmap_type = ttm_bo_map_kmap;
 426                 map->page = ttm_tt_get_page(ttm, start_page);
 427                 map->virtual = kmap(map->page);
 428         } else {
 429             /*
 430              * Populate the part we're mapping;
 431              */
 432                 for (i = start_page; i < start_page + num_pages; ++i) {
 433                         d = ttm_tt_get_page(ttm, i);
 434                         if (!d)
 435                                 return -ENOMEM;
 436                 }
 437
 438                 /*
 439                  * We need to use vmap to get the desired page protection
 440                  * or to make the buffer object look contiguous.
 441                  */
 442                 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
 443                         PAGE_KERNEL :
 444                         ttm_io_prot(mem->placement, PAGE_KERNEL);
 445                 map->bo_kmap_type = ttm_bo_map_vmap;
 446                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 447                                     0, prot);
 448         }
 449         return (!map->virtual) ? -ENOMEM : 0;
 450 }
 451
 452 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 453                 unsigned long start_page, unsigned long num_pages,
 454                 struct ttm_bo_kmap_obj *map)
 455 {
 456         unsigned long offset, size;
 457         int ret;
 458
 459         BUG_ON(!list_empty(&bo->swap));
 460         map->virtual = NULL;
 461         map->bo = bo;
 462         if (num_pages > bo->num_pages)
 463                 return -EINVAL;
 464         if (start_page > bo->num_pages)
 465                 return -EINVAL;
 466 #if 0
 467         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
 468                 return -EPERM;
 469 #endif
 470         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 471         if (ret)
 472                 return ret;
 473         if (!bo->mem.bus.is_iomem) {
 474                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 475         } else {
 476                 offset = start_page << PAGE_SHIFT;
 477                 size = num_pages << PAGE_SHIFT;
 478                 return ttm_bo_ioremap(bo, offset, size, map);
 479         }
 480 }
 481 EXPORT_SYMBOL(ttm_bo_kmap);
 482
 483 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 484 {
 485         if (!map->virtual)
 486                 return;
 487         switch (map->bo_kmap_type) {
 488         case ttm_bo_map_iomap:
 489                 iounmap(map->virtual);
 490                 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 491                 break;
 492         case ttm_bo_map_vmap:
 493                 vunmap(map->virtual);
 494                 break;
 495         case ttm_bo_map_kmap:
 496                 kunmap(map->page);
 497                 break;
 498         case ttm_bo_map_premapped:
 499                 break;
 500         default:
 501                 BUG();
 502         }
 503         map->virtual = NULL;
 504         map->page = NULL;
 505 }
 506 EXPORT_SYMBOL(ttm_bo_kunmap);
 507
 508 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 509                               void *sync_obj,
 510                               void *sync_obj_arg,
 511                               bool evict, bool no_wait_reserve,
 512                               bool no_wait_gpu,
 513                               struct ttm_mem_reg *new_mem)
 514 {
 515         struct ttm_bo_device *bdev = bo->bdev;
 516         struct ttm_bo_driver *driver = bdev->driver;
 517         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 518         struct ttm_mem_reg *old_mem = &bo->mem;
 519         int ret;
 520         struct ttm_buffer_object *ghost_obj;
 521         void *tmp_obj = NULL;
 522
 523         spin_lock(&bo->lock);
 524         if (bo->sync_obj) {
 525                 tmp_obj = bo->sync_obj;
 526                 bo->sync_obj = NULL;
 527         }
 528         bo->sync_obj = driver->sync_obj_ref(sync_obj);
 529         bo->sync_obj_arg = sync_obj_arg;
 530         if (evict) {
 531                 ret = ttm_bo_wait(bo, false, false, false);
 532                 spin_unlock(&bo->lock);
 533                 if (tmp_obj)
 534                         driver->sync_obj_unref(&tmp_obj);
 535                 if (ret)
 536                         return ret;
 537
 538                 ttm_bo_free_old_node(bo);
 539                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 540                     (bo->ttm != NULL)) {
 541                         ttm_tt_unbind(bo->ttm);
 542                         ttm_tt_destroy(bo->ttm);
 543                         bo->ttm = NULL;
 544                 }
 545         } else {
 546                 /**
 547                  * This should help pipeline ordinary buffer moves.
 548                  *
 549                  * Hang old buffer memory on a new buffer object,
 550                  * and leave it to be released when the GPU
 551                  * operation has completed.
 552                  */
 553
 554                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
 555                 spin_unlock(&bo->lock);
 556                 if (tmp_obj)
 557                         driver->sync_obj_unref(&tmp_obj);
 558
 559                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 560                 if (ret)
 561                         return ret;
 562
 563                 /**
 564                  * If we're not moving to fixed memory, the TTM object
 565                  * needs to stay alive. Otherwhise hang it on the ghost
 566                  * bo to be unbound and destroyed.
 567                  */
 568
 569                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 570                         ghost_obj->ttm = NULL;
 571                 else
 572                         bo->ttm = NULL;
 573
 574                 ttm_bo_unreserve(ghost_obj);
 575                 ttm_bo_unref(&ghost_obj);
 576         }
 577
 578         *old_mem = *new_mem;
 579         new_mem->mm_node = NULL;
 580
 581         return 0;
 582 }
 583 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);