drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #include "ttm/ttm_bo_driver.h"
  32 #include "ttm/ttm_placement.h"
  33 #include <linux/io.h>
  34 #include <linux/highmem.h>
  35 #include <linux/wait.h>
  36 #include <linux/slab.h>
  37 #include <linux/vmalloc.h>
  38 #include <linux/module.h>
  39
  40 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  41 {
  42         struct ttm_mem_reg *old_mem = &bo->mem;
  43
  44         if (old_mem->mm_node) {
  45                 spin_lock(&bo->glob->lru_lock);
  46                 drm_mm_put_block(old_mem->mm_node);
  47                 spin_unlock(&bo->glob->lru_lock);
  48         }
  49         old_mem->mm_node = NULL;
  50 }
  51
  52 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  53                     bool evict, bool no_wait_reserve,
  54                     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
  55 {
  56         struct ttm_tt *ttm = bo->ttm;
  57         struct ttm_mem_reg *old_mem = &bo->mem;
  58         int ret;
  59
  60         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  61                 ttm_tt_unbind(ttm);
  62                 ttm_bo_free_old_node(bo);
  63                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  64                                 TTM_PL_MASK_MEM);
  65                 old_mem->mem_type = TTM_PL_SYSTEM;
  66         }
  67
  68         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  69         if (unlikely(ret != 0))
  70                 return ret;
  71
  72         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  73                 ret = ttm_tt_bind(ttm, new_mem);
  74                 if (unlikely(ret != 0))
  75                         return ret;
  76         }
  77
  78         *old_mem = *new_mem;
  79         new_mem->mm_node = NULL;
  80
  81         return 0;
  82 }
  83 EXPORT_SYMBOL(ttm_bo_move_ttm);
  84
  85 int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  86 {
  87         int ret;
  88
  89         if (!mem->bus.io_reserved) {
  90                 mem->bus.io_reserved = true;
  91                 ret = bdev->driver->io_mem_reserve(bdev, mem);
  92                 if (unlikely(ret != 0))
  93                         return ret;
  94         }
  95         return 0;
  96 }
  97
  98 void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  99 {
 100         if (bdev->driver->io_mem_reserve) {
 101                 if (mem->bus.io_reserved) {
 102                         mem->bus.io_reserved = false;
 103                         bdev->driver->io_mem_free(bdev, mem);
 104                 }
 105         }
 106 }
 107
 108 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 109                         void **virtual)
 110 {
 111         int ret;
 112         void *addr;
 113
 114         *virtual = NULL;
 115         ret = ttm_mem_io_reserve(bdev, mem);
 116         if (ret)
 117                 return ret;
 118
 119         if (mem->bus.addr) {
 120                 addr = mem->bus.addr;
 121         } else {
 122                 if (mem->placement & TTM_PL_FLAG_WC)
 123                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 124                 else
 125                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 126                 if (!addr) {
 127                         ttm_mem_io_free(bdev, mem);
 128                         return -ENOMEM;
 129                 }
 130         }
 131         *virtual = addr;
 132         return 0;
 133 }
 134
 135 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 136                          void *virtual)
 137 {
 138         struct ttm_mem_type_manager *man;
 139
 140         man = &bdev->man[mem->mem_type];
 141
 142         if (virtual && mem->bus.addr == NULL)
 143                 iounmap(virtual);
 144         ttm_mem_io_free(bdev, mem);
 145 }
 146
 147 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 148 {
 149         uint32_t *dstP =
 150             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 151         uint32_t *srcP =
 152             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 153
 154         int i;
 155         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 156                 iowrite32(ioread32(srcP++), dstP++);
 157         return 0;
 158 }
 159
 160 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 161                                 unsigned long page,
 162                                 pgprot_t prot)
 163 {
 164         struct page *d = ttm_tt_get_page(ttm, page);
 165         void *dst;
 166
 167         if (!d)
 168                 return -ENOMEM;
 169
 170         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 171
 172 #ifdef CONFIG_X86
 173         dst = kmap_atomic_prot(d, KM_USER0, prot);
 174 #else
 175         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 176                 dst = vmap(&d, 1, 0, prot);
 177         else
 178                 dst = kmap(d);
 179 #endif
 180         if (!dst)
 181                 return -ENOMEM;
 182
 183         memcpy_fromio(dst, src, PAGE_SIZE);
 184
 185 #ifdef CONFIG_X86
 186         kunmap_atomic(dst, KM_USER0);
 187 #else
 188         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 189                 vunmap(dst);
 190         else
 191                 kunmap(d);
 192 #endif
 193
 194         return 0;
 195 }
 196
 197 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 198                                 unsigned long page,
 199                                 pgprot_t prot)
 200 {
 201         struct page *s = ttm_tt_get_page(ttm, page);
 202         void *src;
 203
 204         if (!s)
 205                 return -ENOMEM;
 206
 207         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 208 #ifdef CONFIG_X86
 209         src = kmap_atomic_prot(s, KM_USER0, prot);
 210 #else
 211         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 212                 src = vmap(&s, 1, 0, prot);
 213         else
 214                 src = kmap(s);
 215 #endif
 216         if (!src)
 217                 return -ENOMEM;
 218
 219         memcpy_toio(dst, src, PAGE_SIZE);
 220
 221 #ifdef CONFIG_X86
 222         kunmap_atomic(src, KM_USER0);
 223 #else
 224         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 225                 vunmap(src);
 226         else
 227                 kunmap(s);
 228 #endif
 229
 230         return 0;
 231 }
 232
 233 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 234                        bool evict, bool no_wait_reserve, bool no_wait_gpu,
 235                        struct ttm_mem_reg *new_mem)
 236 {
 237         struct ttm_bo_device *bdev = bo->bdev;
 238         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 239         struct ttm_tt *ttm = bo->ttm;
 240         struct ttm_mem_reg *old_mem = &bo->mem;
 241         struct ttm_mem_reg old_copy = *old_mem;
 242         void *old_iomap;
 243         void *new_iomap;
 244         int ret;
 245         unsigned long i;
 246         unsigned long page;
 247         unsigned long add = 0;
 248         int dir;
 249
 250         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 251         if (ret)
 252                 return ret;
 253         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 254         if (ret)
 255                 goto out;
 256
 257         if (old_iomap == NULL && new_iomap == NULL)
 258                 goto out2;
 259         if (old_iomap == NULL && ttm == NULL)
 260                 goto out2;
 261
 262         add = 0;
 263         dir = 1;
 264
 265         if ((old_mem->mem_type == new_mem->mem_type) &&
 266             (new_mem->mm_node->start <
 267              old_mem->mm_node->start + old_mem->mm_node->size)) {
 268                 dir = -1;
 269                 add = new_mem->num_pages - 1;
 270         }
 271
 272         for (i = 0; i < new_mem->num_pages; ++i) {
 273                 page = i * dir + add;
 274                 if (old_iomap == NULL) {
 275                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 276                                                     PAGE_KERNEL);
 277                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 278                                                    prot);
 279                 } else if (new_iomap == NULL) {
 280                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 281                                                     PAGE_KERNEL);
 282                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 283                                                    prot);
 284                 } else
 285                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 286                 if (ret)
 287                         goto out1;
 288         }
 289         mb();
 290 out2:
 291         ttm_bo_free_old_node(bo);
 292
 293         *old_mem = *new_mem;
 294         new_mem->mm_node = NULL;
 295
 296         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
 297                 ttm_tt_unbind(ttm);
 298                 ttm_tt_destroy(ttm);
 299                 bo->ttm = NULL;
 300         }
 301
 302 out1:
 303         ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
 304 out:
 305         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 306         return ret;
 307 }
 308 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 309
 310 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 311 {
 312         kfree(bo);
 313 }
 314
 315 /**
 316  * ttm_buffer_object_transfer
 317  *
 318  * @bo: A pointer to a struct ttm_buffer_object.
 319  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 320  * holding the data of @bo with the old placement.
 321  *
 322  * This is a utility function that may be called after an accelerated move
 323  * has been scheduled. A new buffer object is created as a placeholder for
 324  * the old data while it's being copied. When that buffer object is idle,
 325  * it can be destroyed, releasing the space of the old placement.
 326  * Returns:
 327  * !0: Failure.
 328  */
 329
 330 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 331                                       struct ttm_buffer_object **new_obj)
 332 {
 333         struct ttm_buffer_object *fbo;
 334         struct ttm_bo_device *bdev = bo->bdev;
 335         struct ttm_bo_driver *driver = bdev->driver;
 336
 337         fbo = kzalloc(sizeof(*fbo), GFP_KERNEL);
 338         if (!fbo)
 339                 return -ENOMEM;
 340
 341         *fbo = *bo;
 342
 343         /**
 344          * Fix up members that we shouldn't copy directly:
 345          * TODO: Explicit member copy would probably be better here.
 346          */
 347
 348         spin_lock_init(&fbo->lock);
 349         init_waitqueue_head(&fbo->event_queue);
 350         INIT_LIST_HEAD(&fbo->ddestroy);
 351         INIT_LIST_HEAD(&fbo->lru);
 352         INIT_LIST_HEAD(&fbo->swap);
 353         fbo->vm_node = NULL;
 354
 355         fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
 356         if (fbo->mem.mm_node)
 357                 fbo->mem.mm_node->private = (void *)fbo;
 358         kref_init(&fbo->list_kref);
 359         kref_init(&fbo->kref);
 360         fbo->destroy = &ttm_transfered_destroy;
 361
 362         *new_obj = fbo;
 363         return 0;
 364 }
 365
 366 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 367 {
 368 #if defined(__i386__) || defined(__x86_64__)
 369         if (caching_flags & TTM_PL_FLAG_WC)
 370                 tmp = pgprot_writecombine(tmp);
 371         else if (boot_cpu_data.x86 > 3)
 372                 tmp = pgprot_noncached(tmp);
 373
 374 #elif defined(__powerpc__)
 375         if (!(caching_flags & TTM_PL_FLAG_CACHED)) {
 376                 pgprot_val(tmp) |= _PAGE_NO_CACHE;
 377                 if (caching_flags & TTM_PL_FLAG_UNCACHED)
 378                         pgprot_val(tmp) |= _PAGE_GUARDED;
 379         }
 380 #endif
 381 #if defined(__ia64__)
 382         if (caching_flags & TTM_PL_FLAG_WC)
 383                 tmp = pgprot_writecombine(tmp);
 384         else
 385                 tmp = pgprot_noncached(tmp);
 386 #endif
 387 #if defined(__sparc__)
 388         if (!(caching_flags & TTM_PL_FLAG_CACHED))
 389                 tmp = pgprot_noncached(tmp);
 390 #endif
 391         return tmp;
 392 }
 393 EXPORT_SYMBOL(ttm_io_prot);
 394
 395 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 396                           unsigned long offset,
 397                           unsigned long size,
 398                           struct ttm_bo_kmap_obj *map)
 399 {
 400         struct ttm_mem_reg *mem = &bo->mem;
 401
 402         if (bo->mem.bus.addr) {
 403                 map->bo_kmap_type = ttm_bo_map_premapped;
 404                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 405         } else {
 406                 map->bo_kmap_type = ttm_bo_map_iomap;
 407                 if (mem->placement & TTM_PL_FLAG_WC)
 408                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 409                                                   size);
 410                 else
 411                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 412                                                        size);
 413         }
 414         return (!map->virtual) ? -ENOMEM : 0;
 415 }
 416
 417 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 418                            unsigned long start_page,
 419                            unsigned long num_pages,
 420                            struct ttm_bo_kmap_obj *map)
 421 {
 422         struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
 423         struct ttm_tt *ttm = bo->ttm;
 424         struct page *d;
 425         int i;
 426
 427         BUG_ON(!ttm);
 428         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 429                 /*
 430                  * We're mapping a single page, and the desired
 431                  * page protection is consistent with the bo.
 432                  */
 433
 434                 map->bo_kmap_type = ttm_bo_map_kmap;
 435                 map->page = ttm_tt_get_page(ttm, start_page);
 436                 map->virtual = kmap(map->page);
 437         } else {
 438             /*
 439              * Populate the part we're mapping;
 440              */
 441                 for (i = start_page; i < start_page + num_pages; ++i) {
 442                         d = ttm_tt_get_page(ttm, i);
 443                         if (!d)
 444                                 return -ENOMEM;
 445                 }
 446
 447                 /*
 448                  * We need to use vmap to get the desired page protection
 449                  * or to make the buffer object look contiguous.
 450                  */
 451                 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
 452                         PAGE_KERNEL :
 453                         ttm_io_prot(mem->placement, PAGE_KERNEL);
 454                 map->bo_kmap_type = ttm_bo_map_vmap;
 455                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 456                                     0, prot);
 457         }
 458         return (!map->virtual) ? -ENOMEM : 0;
 459 }
 460
 461 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 462                 unsigned long start_page, unsigned long num_pages,
 463                 struct ttm_bo_kmap_obj *map)
 464 {
 465         unsigned long offset, size;
 466         int ret;
 467
 468         BUG_ON(!list_empty(&bo->swap));
 469         map->virtual = NULL;
 470         map->bo = bo;
 471         if (num_pages > bo->num_pages)
 472                 return -EINVAL;
 473         if (start_page > bo->num_pages)
 474                 return -EINVAL;
 475 #if 0
 476         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
 477                 return -EPERM;
 478 #endif
 479         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 480         if (ret)
 481                 return ret;
 482         if (!bo->mem.bus.is_iomem) {
 483                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 484         } else {
 485                 offset = start_page << PAGE_SHIFT;
 486                 size = num_pages << PAGE_SHIFT;
 487                 return ttm_bo_ioremap(bo, offset, size, map);
 488         }
 489 }
 490 EXPORT_SYMBOL(ttm_bo_kmap);
 491
 492 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 493 {
 494         if (!map->virtual)
 495                 return;
 496         switch (map->bo_kmap_type) {
 497         case ttm_bo_map_iomap:
 498                 iounmap(map->virtual);
 499                 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 500                 break;
 501         case ttm_bo_map_vmap:
 502                 vunmap(map->virtual);
 503                 break;
 504         case ttm_bo_map_kmap:
 505                 kunmap(map->page);
 506                 break;
 507         case ttm_bo_map_premapped:
 508                 break;
 509         default:
 510                 BUG();
 511         }
 512         map->virtual = NULL;
 513         map->page = NULL;
 514 }
 515 EXPORT_SYMBOL(ttm_bo_kunmap);
 516
 517 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 518                               void *sync_obj,
 519                               void *sync_obj_arg,
 520                               bool evict, bool no_wait_reserve,
 521                               bool no_wait_gpu,
 522                               struct ttm_mem_reg *new_mem)
 523 {
 524         struct ttm_bo_device *bdev = bo->bdev;
 525         struct ttm_bo_driver *driver = bdev->driver;
 526         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 527         struct ttm_mem_reg *old_mem = &bo->mem;
 528         int ret;
 529         struct ttm_buffer_object *ghost_obj;
 530         void *tmp_obj = NULL;
 531
 532         spin_lock(&bo->lock);
 533         if (bo->sync_obj) {
 534                 tmp_obj = bo->sync_obj;
 535                 bo->sync_obj = NULL;
 536         }
 537         bo->sync_obj = driver->sync_obj_ref(sync_obj);
 538         bo->sync_obj_arg = sync_obj_arg;
 539         if (evict) {
 540                 ret = ttm_bo_wait(bo, false, false, false);
 541                 spin_unlock(&bo->lock);
 542                 if (tmp_obj)
 543                         driver->sync_obj_unref(&tmp_obj);
 544                 if (ret)
 545                         return ret;
 546
 547                 ttm_bo_free_old_node(bo);
 548                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 549                     (bo->ttm != NULL)) {
 550                         ttm_tt_unbind(bo->ttm);
 551                         ttm_tt_destroy(bo->ttm);
 552                         bo->ttm = NULL;
 553                 }
 554         } else {
 555                 /**
 556                  * This should help pipeline ordinary buffer moves.
 557                  *
 558                  * Hang old buffer memory on a new buffer object,
 559                  * and leave it to be released when the GPU
 560                  * operation has completed.
 561                  */
 562
 563                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
 564                 spin_unlock(&bo->lock);
 565                 if (tmp_obj)
 566                         driver->sync_obj_unref(&tmp_obj);
 567
 568                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 569                 if (ret)
 570                         return ret;
 571
 572                 /**
 573                  * If we're not moving to fixed memory, the TTM object
 574                  * needs to stay alive. Otherwhise hang it on the ghost
 575                  * bo to be unbound and destroyed.
 576                  */
 577
 578                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 579                         ghost_obj->ttm = NULL;
 580                 else
 581                         bo->ttm = NULL;
 582
 583                 ttm_bo_unreserve(ghost_obj);
 584                 ttm_bo_unref(&ghost_obj);
 585         }
 586
 587         *old_mem = *new_mem;
 588         new_mem->mm_node = NULL;
 589
 590         return 0;
 591 }
 592 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);