drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #include "ttm/ttm_bo_driver.h"
  32 #include "ttm/ttm_placement.h"
  33 #include <linux/io.h>
  34 #include <linux/highmem.h>
  35 #include <linux/wait.h>
  36 #include <linux/vmalloc.h>
  37 #include <linux/module.h>
  38
  39 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  40 {
  41         struct ttm_mem_reg *old_mem = &bo->mem;
  42
  43         if (old_mem->mm_node) {
  44                 spin_lock(&bo->glob->lru_lock);
  45                 drm_mm_put_block(old_mem->mm_node);
  46                 spin_unlock(&bo->glob->lru_lock);
  47         }
  48         old_mem->mm_node = NULL;
  49 }
  50
  51 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  52                     bool evict, bool no_wait_reserve,
  53                     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
  54 {
  55         struct ttm_tt *ttm = bo->ttm;
  56         struct ttm_mem_reg *old_mem = &bo->mem;
  57         int ret;
  58
  59         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  60                 ttm_tt_unbind(ttm);
  61                 ttm_bo_free_old_node(bo);
  62                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  63                                 TTM_PL_MASK_MEM);
  64                 old_mem->mem_type = TTM_PL_SYSTEM;
  65         }
  66
  67         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  68         if (unlikely(ret != 0))
  69                 return ret;
  70
  71         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  72                 ret = ttm_tt_bind(ttm, new_mem);
  73                 if (unlikely(ret != 0))
  74                         return ret;
  75         }
  76
  77         *old_mem = *new_mem;
  78         new_mem->mm_node = NULL;
  79
  80         return 0;
  81 }
  82 EXPORT_SYMBOL(ttm_bo_move_ttm);
  83
  84 int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  85 {
  86         int ret;
  87
  88         if (!mem->bus.io_reserved) {
  89                 mem->bus.io_reserved = true;
  90                 ret = bdev->driver->io_mem_reserve(bdev, mem);
  91                 if (unlikely(ret != 0))
  92                         return ret;
  93         }
  94         return 0;
  95 }
  96
  97 void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  98 {
  99         if (bdev->driver->io_mem_reserve) {
 100                 if (mem->bus.io_reserved) {
 101                         mem->bus.io_reserved = false;
 102                         bdev->driver->io_mem_free(bdev, mem);
 103                 }
 104         }
 105 }
 106
 107 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 108                         void **virtual)
 109 {
 110         int ret;
 111         void *addr;
 112
 113         *virtual = NULL;
 114         ret = ttm_mem_io_reserve(bdev, mem);
 115         if (ret)
 116                 return ret;
 117
 118         if (mem->bus.addr) {
 119                 addr = mem->bus.addr;
 120         } else {
 121                 if (mem->placement & TTM_PL_FLAG_WC)
 122                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 123                 else
 124                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 125                 if (!addr) {
 126                         ttm_mem_io_free(bdev, mem);
 127                         return -ENOMEM;
 128                 }
 129         }
 130         *virtual = addr;
 131         return 0;
 132 }
 133
 134 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 135                          void *virtual)
 136 {
 137         struct ttm_mem_type_manager *man;
 138
 139         man = &bdev->man[mem->mem_type];
 140
 141         if (virtual && mem->bus.addr == NULL)
 142                 iounmap(virtual);
 143         ttm_mem_io_free(bdev, mem);
 144 }
 145
 146 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 147 {
 148         uint32_t *dstP =
 149             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 150         uint32_t *srcP =
 151             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 152
 153         int i;
 154         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 155                 iowrite32(ioread32(srcP++), dstP++);
 156         return 0;
 157 }
 158
 159 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 160                                 unsigned long page,
 161                                 pgprot_t prot)
 162 {
 163         struct page *d = ttm_tt_get_page(ttm, page);
 164         void *dst;
 165
 166         if (!d)
 167                 return -ENOMEM;
 168
 169         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 170
 171 #ifdef CONFIG_X86
 172         dst = kmap_atomic_prot(d, KM_USER0, prot);
 173 #else
 174         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 175                 dst = vmap(&d, 1, 0, prot);
 176         else
 177                 dst = kmap(d);
 178 #endif
 179         if (!dst)
 180                 return -ENOMEM;
 181
 182         memcpy_fromio(dst, src, PAGE_SIZE);
 183
 184 #ifdef CONFIG_X86
 185         kunmap_atomic(dst, KM_USER0);
 186 #else
 187         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 188                 vunmap(dst);
 189         else
 190                 kunmap(d);
 191 #endif
 192
 193         return 0;
 194 }
 195
 196 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 197                                 unsigned long page,
 198                                 pgprot_t prot)
 199 {
 200         struct page *s = ttm_tt_get_page(ttm, page);
 201         void *src;
 202
 203         if (!s)
 204                 return -ENOMEM;
 205
 206         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 207 #ifdef CONFIG_X86
 208         src = kmap_atomic_prot(s, KM_USER0, prot);
 209 #else
 210         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 211                 src = vmap(&s, 1, 0, prot);
 212         else
 213                 src = kmap(s);
 214 #endif
 215         if (!src)
 216                 return -ENOMEM;
 217
 218         memcpy_toio(dst, src, PAGE_SIZE);
 219
 220 #ifdef CONFIG_X86
 221         kunmap_atomic(src, KM_USER0);
 222 #else
 223         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 224                 vunmap(src);
 225         else
 226                 kunmap(s);
 227 #endif
 228
 229         return 0;
 230 }
 231
 232 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 233                        bool evict, bool no_wait_reserve, bool no_wait_gpu,
 234                        struct ttm_mem_reg *new_mem)
 235 {
 236         struct ttm_bo_device *bdev = bo->bdev;
 237         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 238         struct ttm_tt *ttm = bo->ttm;
 239         struct ttm_mem_reg *old_mem = &bo->mem;
 240         struct ttm_mem_reg old_copy = *old_mem;
 241         void *old_iomap;
 242         void *new_iomap;
 243         int ret;
 244         unsigned long i;
 245         unsigned long page;
 246         unsigned long add = 0;
 247         int dir;
 248
 249         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 250         if (ret)
 251                 return ret;
 252         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 253         if (ret)
 254                 goto out;
 255
 256         if (old_iomap == NULL && new_iomap == NULL)
 257                 goto out2;
 258         if (old_iomap == NULL && ttm == NULL)
 259                 goto out2;
 260
 261         add = 0;
 262         dir = 1;
 263
 264         if ((old_mem->mem_type == new_mem->mem_type) &&
 265             (new_mem->mm_node->start <
 266              old_mem->mm_node->start + old_mem->mm_node->size)) {
 267                 dir = -1;
 268                 add = new_mem->num_pages - 1;
 269         }
 270
 271         for (i = 0; i < new_mem->num_pages; ++i) {
 272                 page = i * dir + add;
 273                 if (old_iomap == NULL) {
 274                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 275                                                     PAGE_KERNEL);
 276                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 277                                                    prot);
 278                 } else if (new_iomap == NULL) {
 279                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 280                                                     PAGE_KERNEL);
 281                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 282                                                    prot);
 283                 } else
 284                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 285                 if (ret)
 286                         goto out1;
 287         }
 288         mb();
 289 out2:
 290         ttm_bo_free_old_node(bo);
 291
 292         *old_mem = *new_mem;
 293         new_mem->mm_node = NULL;
 294
 295         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
 296                 ttm_tt_unbind(ttm);
 297                 ttm_tt_destroy(ttm);
 298                 bo->ttm = NULL;
 299         }
 300
 301 out1:
 302         ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
 303 out:
 304         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 305         return ret;
 306 }
 307 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 308
 309 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 310 {
 311         kfree(bo);
 312 }
 313
 314 /**
 315  * ttm_buffer_object_transfer
 316  *
 317  * @bo: A pointer to a struct ttm_buffer_object.
 318  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 319  * holding the data of @bo with the old placement.
 320  *
 321  * This is a utility function that may be called after an accelerated move
 322  * has been scheduled. A new buffer object is created as a placeholder for
 323  * the old data while it's being copied. When that buffer object is idle,
 324  * it can be destroyed, releasing the space of the old placement.
 325  * Returns:
 326  * !0: Failure.
 327  */
 328
 329 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 330                                       struct ttm_buffer_object **new_obj)
 331 {
 332         struct ttm_buffer_object *fbo;
 333         struct ttm_bo_device *bdev = bo->bdev;
 334         struct ttm_bo_driver *driver = bdev->driver;
 335
 336         fbo = kzalloc(sizeof(*fbo), GFP_KERNEL);
 337         if (!fbo)
 338                 return -ENOMEM;
 339
 340         *fbo = *bo;
 341
 342         /**
 343          * Fix up members that we shouldn't copy directly:
 344          * TODO: Explicit member copy would probably be better here.
 345          */
 346
 347         spin_lock_init(&fbo->lock);
 348         init_waitqueue_head(&fbo->event_queue);
 349         INIT_LIST_HEAD(&fbo->ddestroy);
 350         INIT_LIST_HEAD(&fbo->lru);
 351         INIT_LIST_HEAD(&fbo->swap);
 352         fbo->vm_node = NULL;
 353
 354         fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
 355         if (fbo->mem.mm_node)
 356                 fbo->mem.mm_node->private = (void *)fbo;
 357         kref_init(&fbo->list_kref);
 358         kref_init(&fbo->kref);
 359         fbo->destroy = &ttm_transfered_destroy;
 360
 361         *new_obj = fbo;
 362         return 0;
 363 }
 364
 365 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 366 {
 367 #if defined(__i386__) || defined(__x86_64__)
 368         if (caching_flags & TTM_PL_FLAG_WC)
 369                 tmp = pgprot_writecombine(tmp);
 370         else if (boot_cpu_data.x86 > 3)
 371                 tmp = pgprot_noncached(tmp);
 372
 373 #elif defined(__powerpc__)
 374         if (!(caching_flags & TTM_PL_FLAG_CACHED)) {
 375                 pgprot_val(tmp) |= _PAGE_NO_CACHE;
 376                 if (caching_flags & TTM_PL_FLAG_UNCACHED)
 377                         pgprot_val(tmp) |= _PAGE_GUARDED;
 378         }
 379 #endif
 380 #if defined(__ia64__)
 381         if (caching_flags & TTM_PL_FLAG_WC)
 382                 tmp = pgprot_writecombine(tmp);
 383         else
 384                 tmp = pgprot_noncached(tmp);
 385 #endif
 386 #if defined(__sparc__)
 387         if (!(caching_flags & TTM_PL_FLAG_CACHED))
 388                 tmp = pgprot_noncached(tmp);
 389 #endif
 390         return tmp;
 391 }
 392 EXPORT_SYMBOL(ttm_io_prot);
 393
 394 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 395                           unsigned long offset,
 396                           unsigned long size,
 397                           struct ttm_bo_kmap_obj *map)
 398 {
 399         struct ttm_mem_reg *mem = &bo->mem;
 400
 401         if (bo->mem.bus.addr) {
 402                 map->bo_kmap_type = ttm_bo_map_premapped;
 403                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 404         } else {
 405                 map->bo_kmap_type = ttm_bo_map_iomap;
 406                 if (mem->placement & TTM_PL_FLAG_WC)
 407                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 408                                                   size);
 409                 else
 410                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 411                                                        size);
 412         }
 413         return (!map->virtual) ? -ENOMEM : 0;
 414 }
 415
 416 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 417                            unsigned long start_page,
 418                            unsigned long num_pages,
 419                            struct ttm_bo_kmap_obj *map)
 420 {
 421         struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
 422         struct ttm_tt *ttm = bo->ttm;
 423         struct page *d;
 424         int i;
 425
 426         BUG_ON(!ttm);
 427         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 428                 /*
 429                  * We're mapping a single page, and the desired
 430                  * page protection is consistent with the bo.
 431                  */
 432
 433                 map->bo_kmap_type = ttm_bo_map_kmap;
 434                 map->page = ttm_tt_get_page(ttm, start_page);
 435                 map->virtual = kmap(map->page);
 436         } else {
 437             /*
 438              * Populate the part we're mapping;
 439              */
 440                 for (i = start_page; i < start_page + num_pages; ++i) {
 441                         d = ttm_tt_get_page(ttm, i);
 442                         if (!d)
 443                                 return -ENOMEM;
 444                 }
 445
 446                 /*
 447                  * We need to use vmap to get the desired page protection
 448                  * or to make the buffer object look contiguous.
 449                  */
 450                 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
 451                         PAGE_KERNEL :
 452                         ttm_io_prot(mem->placement, PAGE_KERNEL);
 453                 map->bo_kmap_type = ttm_bo_map_vmap;
 454                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 455                                     0, prot);
 456         }
 457         return (!map->virtual) ? -ENOMEM : 0;
 458 }
 459
 460 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 461                 unsigned long start_page, unsigned long num_pages,
 462                 struct ttm_bo_kmap_obj *map)
 463 {
 464         unsigned long offset, size;
 465         int ret;
 466
 467         BUG_ON(!list_empty(&bo->swap));
 468         map->virtual = NULL;
 469         map->bo = bo;
 470         if (num_pages > bo->num_pages)
 471                 return -EINVAL;
 472         if (start_page > bo->num_pages)
 473                 return -EINVAL;
 474 #if 0
 475         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
 476                 return -EPERM;
 477 #endif
 478         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 479         if (ret)
 480                 return ret;
 481         if (!bo->mem.bus.is_iomem) {
 482                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 483         } else {
 484                 offset = start_page << PAGE_SHIFT;
 485                 size = num_pages << PAGE_SHIFT;
 486                 return ttm_bo_ioremap(bo, offset, size, map);
 487         }
 488 }
 489 EXPORT_SYMBOL(ttm_bo_kmap);
 490
 491 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 492 {
 493         if (!map->virtual)
 494                 return;
 495         switch (map->bo_kmap_type) {
 496         case ttm_bo_map_iomap:
 497                 iounmap(map->virtual);
 498                 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 499                 break;
 500         case ttm_bo_map_vmap:
 501                 vunmap(map->virtual);
 502                 break;
 503         case ttm_bo_map_kmap:
 504                 kunmap(map->page);
 505                 break;
 506         case ttm_bo_map_premapped:
 507                 break;
 508         default:
 509                 BUG();
 510         }
 511         map->virtual = NULL;
 512         map->page = NULL;
 513 }
 514 EXPORT_SYMBOL(ttm_bo_kunmap);
 515
 516 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 517                               void *sync_obj,
 518                               void *sync_obj_arg,
 519                               bool evict, bool no_wait_reserve,
 520                               bool no_wait_gpu,
 521                               struct ttm_mem_reg *new_mem)
 522 {
 523         struct ttm_bo_device *bdev = bo->bdev;
 524         struct ttm_bo_driver *driver = bdev->driver;
 525         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 526         struct ttm_mem_reg *old_mem = &bo->mem;
 527         int ret;
 528         struct ttm_buffer_object *ghost_obj;
 529         void *tmp_obj = NULL;
 530
 531         spin_lock(&bo->lock);
 532         if (bo->sync_obj) {
 533                 tmp_obj = bo->sync_obj;
 534                 bo->sync_obj = NULL;
 535         }
 536         bo->sync_obj = driver->sync_obj_ref(sync_obj);
 537         bo->sync_obj_arg = sync_obj_arg;
 538         if (evict) {
 539                 ret = ttm_bo_wait(bo, false, false, false);
 540                 spin_unlock(&bo->lock);
 541                 if (tmp_obj)
 542                         driver->sync_obj_unref(&tmp_obj);
 543                 if (ret)
 544                         return ret;
 545
 546                 ttm_bo_free_old_node(bo);
 547                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 548                     (bo->ttm != NULL)) {
 549                         ttm_tt_unbind(bo->ttm);
 550                         ttm_tt_destroy(bo->ttm);
 551                         bo->ttm = NULL;
 552                 }
 553         } else {
 554                 /**
 555                  * This should help pipeline ordinary buffer moves.
 556                  *
 557                  * Hang old buffer memory on a new buffer object,
 558                  * and leave it to be released when the GPU
 559                  * operation has completed.
 560                  */
 561
 562                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
 563                 spin_unlock(&bo->lock);
 564                 if (tmp_obj)
 565                         driver->sync_obj_unref(&tmp_obj);
 566
 567                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 568                 if (ret)
 569                         return ret;
 570
 571                 /**
 572                  * If we're not moving to fixed memory, the TTM object
 573                  * needs to stay alive. Otherwhise hang it on the ghost
 574                  * bo to be unbound and destroyed.
 575                  */
 576
 577                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 578                         ghost_obj->ttm = NULL;
 579                 else
 580                         bo->ttm = NULL;
 581
 582                 ttm_bo_unreserve(ghost_obj);
 583                 ttm_bo_unref(&ghost_obj);
 584         }
 585
 586         *old_mem = *new_mem;
 587         new_mem->mm_node = NULL;
 588
 589         return 0;
 590 }
 591 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);