drivers/gpu/drm/ttm/ttm_bo_util.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30
  31 #include "ttm/ttm_bo_driver.h"
  32 #include "ttm/ttm_placement.h"
  33 #include <linux/io.h>
  34 #include <linux/highmem.h>
  35 #include <linux/wait.h>
  36 #include <linux/slab.h>
  37 #include <linux/vmalloc.h>
  38 #include <linux/module.h>
  39
  40 void ttm_bo_free_old_node(struct ttm_buffer_object *bo)
  41 {
  42         struct ttm_mem_reg *old_mem = &bo->mem;
  43
  44         if (old_mem->mm_node) {
  45                 spin_lock(&bo->glob->lru_lock);
  46                 drm_mm_put_block(old_mem->mm_node);
  47                 spin_unlock(&bo->glob->lru_lock);
  48         }
  49         old_mem->mm_node = NULL;
  50 }
  51
  52 int ttm_bo_move_ttm(struct ttm_buffer_object *bo,
  53                     bool evict, bool no_wait_reserve,
  54                     bool no_wait_gpu, struct ttm_mem_reg *new_mem)
  55 {
  56         struct ttm_tt *ttm = bo->ttm;
  57         struct ttm_mem_reg *old_mem = &bo->mem;
  58         int ret;
  59
  60         if (old_mem->mem_type != TTM_PL_SYSTEM) {
  61                 ttm_tt_unbind(ttm);
  62                 ttm_bo_free_old_node(bo);
  63                 ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM,
  64                                 TTM_PL_MASK_MEM);
  65                 old_mem->mem_type = TTM_PL_SYSTEM;
  66         }
  67
  68         ret = ttm_tt_set_placement_caching(ttm, new_mem->placement);
  69         if (unlikely(ret != 0))
  70                 return ret;
  71
  72         if (new_mem->mem_type != TTM_PL_SYSTEM) {
  73                 ret = ttm_tt_bind(ttm, new_mem);
  74                 if (unlikely(ret != 0))
  75                         return ret;
  76         }
  77
  78         *old_mem = *new_mem;
  79         new_mem->mm_node = NULL;
  80
  81         return 0;
  82 }
  83 EXPORT_SYMBOL(ttm_bo_move_ttm);
  84
  85 int ttm_mem_io_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  86 {
  87         int ret;
  88
  89         if (!mem->bus.io_reserved) {
  90                 mem->bus.io_reserved = true;
  91                 ret = bdev->driver->io_mem_reserve(bdev, mem);
  92                 if (unlikely(ret != 0))
  93                         return ret;
  94         }
  95         return 0;
  96 }
  97
  98 void ttm_mem_io_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
  99 {
 100         if (bdev->driver->io_mem_reserve) {
 101                 if (mem->bus.io_reserved) {
 102                         mem->bus.io_reserved = false;
 103                         bdev->driver->io_mem_free(bdev, mem);
 104                 }
 105         }
 106 }
 107
 108 int ttm_mem_reg_ioremap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 109                         void **virtual)
 110 {
 111         int ret;
 112         void *addr;
 113
 114         *virtual = NULL;
 115         ret = ttm_mem_io_reserve(bdev, mem);
 116         if (ret || !mem->bus.is_iomem)
 117                 return ret;
 118
 119         if (mem->bus.addr) {
 120                 addr = mem->bus.addr;
 121         } else {
 122                 if (mem->placement & TTM_PL_FLAG_WC)
 123                         addr = ioremap_wc(mem->bus.base + mem->bus.offset, mem->bus.size);
 124                 else
 125                         addr = ioremap_nocache(mem->bus.base + mem->bus.offset, mem->bus.size);
 126                 if (!addr) {
 127                         ttm_mem_io_free(bdev, mem);
 128                         return -ENOMEM;
 129                 }
 130         }
 131         *virtual = addr;
 132         return 0;
 133 }
 134
 135 void ttm_mem_reg_iounmap(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem,
 136                          void *virtual)
 137 {
 138         struct ttm_mem_type_manager *man;
 139
 140         man = &bdev->man[mem->mem_type];
 141
 142         if (virtual && mem->bus.addr == NULL)
 143                 iounmap(virtual);
 144         ttm_mem_io_free(bdev, mem);
 145 }
 146
 147 static int ttm_copy_io_page(void *dst, void *src, unsigned long page)
 148 {
 149         uint32_t *dstP =
 150             (uint32_t *) ((unsigned long)dst + (page << PAGE_SHIFT));
 151         uint32_t *srcP =
 152             (uint32_t *) ((unsigned long)src + (page << PAGE_SHIFT));
 153
 154         int i;
 155         for (i = 0; i < PAGE_SIZE / sizeof(uint32_t); ++i)
 156                 iowrite32(ioread32(srcP++), dstP++);
 157         return 0;
 158 }
 159
 160 static int ttm_copy_io_ttm_page(struct ttm_tt *ttm, void *src,
 161                                 unsigned long page,
 162                                 pgprot_t prot)
 163 {
 164         struct page *d = ttm_tt_get_page(ttm, page);
 165         void *dst;
 166
 167         if (!d)
 168                 return -ENOMEM;
 169
 170         src = (void *)((unsigned long)src + (page << PAGE_SHIFT));
 171
 172 #ifdef CONFIG_X86
 173         dst = kmap_atomic_prot(d, KM_USER0, prot);
 174 #else
 175         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 176                 dst = vmap(&d, 1, 0, prot);
 177         else
 178                 dst = kmap(d);
 179 #endif
 180         if (!dst)
 181                 return -ENOMEM;
 182
 183         memcpy_fromio(dst, src, PAGE_SIZE);
 184
 185 #ifdef CONFIG_X86
 186         kunmap_atomic(dst, KM_USER0);
 187 #else
 188         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 189                 vunmap(dst);
 190         else
 191                 kunmap(d);
 192 #endif
 193
 194         return 0;
 195 }
 196
 197 static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 198                                 unsigned long page,
 199                                 pgprot_t prot)
 200 {
 201         struct page *s = ttm_tt_get_page(ttm, page);
 202         void *src;
 203
 204         if (!s)
 205                 return -ENOMEM;
 206
 207         dst = (void *)((unsigned long)dst + (page << PAGE_SHIFT));
 208 #ifdef CONFIG_X86
 209         src = kmap_atomic_prot(s, KM_USER0, prot);
 210 #else
 211         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 212                 src = vmap(&s, 1, 0, prot);
 213         else
 214                 src = kmap(s);
 215 #endif
 216         if (!src)
 217                 return -ENOMEM;
 218
 219         memcpy_toio(dst, src, PAGE_SIZE);
 220
 221 #ifdef CONFIG_X86
 222         kunmap_atomic(src, KM_USER0);
 223 #else
 224         if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL))
 225                 vunmap(src);
 226         else
 227                 kunmap(s);
 228 #endif
 229
 230         return 0;
 231 }
 232
 233 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 234                        bool evict, bool no_wait_reserve, bool no_wait_gpu,
 235                        struct ttm_mem_reg *new_mem)
 236 {
 237         struct ttm_bo_device *bdev = bo->bdev;
 238         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 239         struct ttm_tt *ttm = bo->ttm;
 240         struct ttm_mem_reg *old_mem = &bo->mem;
 241         struct ttm_mem_reg old_copy = *old_mem;
 242         void *old_iomap;
 243         void *new_iomap;
 244         int ret;
 245         unsigned long i;
 246         unsigned long page;
 247         unsigned long add = 0;
 248         int dir;
 249
 250         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
 251         if (ret)
 252                 return ret;
 253         ret = ttm_mem_reg_ioremap(bdev, new_mem, &new_iomap);
 254         if (ret)
 255                 goto out;
 256
 257         if (old_iomap == NULL && new_iomap == NULL)
 258                 goto out2;
 259         if (old_iomap == NULL && ttm == NULL)
 260                 goto out2;
 261
 262         add = 0;
 263         dir = 1;
 264
 265         if ((old_mem->mem_type == new_mem->mem_type) &&
 266             (new_mem->mm_node->start <
 267              old_mem->mm_node->start + old_mem->mm_node->size)) {
 268                 dir = -1;
 269                 add = new_mem->num_pages - 1;
 270         }
 271
 272         for (i = 0; i < new_mem->num_pages; ++i) {
 273                 page = i * dir + add;
 274                 if (old_iomap == NULL) {
 275                         pgprot_t prot = ttm_io_prot(old_mem->placement,
 276                                                     PAGE_KERNEL);
 277                         ret = ttm_copy_ttm_io_page(ttm, new_iomap, page,
 278                                                    prot);
 279                 } else if (new_iomap == NULL) {
 280                         pgprot_t prot = ttm_io_prot(new_mem->placement,
 281                                                     PAGE_KERNEL);
 282                         ret = ttm_copy_io_ttm_page(ttm, old_iomap, page,
 283                                                    prot);
 284                 } else
 285                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
 286                 if (ret)
 287                         goto out1;
 288         }
 289         mb();
 290 out2:
 291         ttm_bo_free_old_node(bo);
 292
 293         *old_mem = *new_mem;
 294         new_mem->mm_node = NULL;
 295
 296         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
 297                 ttm_tt_unbind(ttm);
 298                 ttm_tt_destroy(ttm);
 299                 bo->ttm = NULL;
 300         }
 301
 302 out1:
 303         ttm_mem_reg_iounmap(bdev, new_mem, new_iomap);
 304 out:
 305         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
 306         return ret;
 307 }
 308 EXPORT_SYMBOL(ttm_bo_move_memcpy);
 309
 310 static void ttm_transfered_destroy(struct ttm_buffer_object *bo)
 311 {
 312         kfree(bo);
 313 }
 314
 315 /**
 316  * ttm_buffer_object_transfer
 317  *
 318  * @bo: A pointer to a struct ttm_buffer_object.
 319  * @new_obj: A pointer to a pointer to a newly created ttm_buffer_object,
 320  * holding the data of @bo with the old placement.
 321  *
 322  * This is a utility function that may be called after an accelerated move
 323  * has been scheduled. A new buffer object is created as a placeholder for
 324  * the old data while it's being copied. When that buffer object is idle,
 325  * it can be destroyed, releasing the space of the old placement.
 326  * Returns:
 327  * !0: Failure.
 328  */
 329
 330 static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
 331                                       struct ttm_buffer_object **new_obj)
 332 {
 333         struct ttm_buffer_object *fbo;
 334         struct ttm_bo_device *bdev = bo->bdev;
 335         struct ttm_bo_driver *driver = bdev->driver;
 336
 337         fbo = kzalloc(sizeof(*fbo), GFP_KERNEL);
 338         if (!fbo)
 339                 return -ENOMEM;
 340
 341         *fbo = *bo;
 342
 343         /**
 344          * Fix up members that we shouldn't copy directly:
 345          * TODO: Explicit member copy would probably be better here.
 346          */
 347
 348         spin_lock_init(&fbo->lock);
 349         init_waitqueue_head(&fbo->event_queue);
 350         INIT_LIST_HEAD(&fbo->ddestroy);
 351         INIT_LIST_HEAD(&fbo->lru);
 352         INIT_LIST_HEAD(&fbo->swap);
 353         fbo->vm_node = NULL;
 354         atomic_set(&fbo->cpu_writers, 0);
 355
 356         fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj);
 357         kref_init(&fbo->list_kref);
 358         kref_init(&fbo->kref);
 359         fbo->destroy = &ttm_transfered_destroy;
 360
 361         *new_obj = fbo;
 362         return 0;
 363 }
 364
 365 pgprot_t ttm_io_prot(uint32_t caching_flags, pgprot_t tmp)
 366 {
 367 #if defined(__i386__) || defined(__x86_64__)
 368         if (caching_flags & TTM_PL_FLAG_WC)
 369                 tmp = pgprot_writecombine(tmp);
 370         else if (boot_cpu_data.x86 > 3)
 371                 tmp = pgprot_noncached(tmp);
 372
 373 #elif defined(__powerpc__)
 374         if (!(caching_flags & TTM_PL_FLAG_CACHED)) {
 375                 pgprot_val(tmp) |= _PAGE_NO_CACHE;
 376                 if (caching_flags & TTM_PL_FLAG_UNCACHED)
 377                         pgprot_val(tmp) |= _PAGE_GUARDED;
 378         }
 379 #endif
 380 #if defined(__ia64__)
 381         if (caching_flags & TTM_PL_FLAG_WC)
 382                 tmp = pgprot_writecombine(tmp);
 383         else
 384                 tmp = pgprot_noncached(tmp);
 385 #endif
 386 #if defined(__sparc__)
 387         if (!(caching_flags & TTM_PL_FLAG_CACHED))
 388                 tmp = pgprot_noncached(tmp);
 389 #endif
 390         return tmp;
 391 }
 392 EXPORT_SYMBOL(ttm_io_prot);
 393
 394 static int ttm_bo_ioremap(struct ttm_buffer_object *bo,
 395                           unsigned long offset,
 396                           unsigned long size,
 397                           struct ttm_bo_kmap_obj *map)
 398 {
 399         struct ttm_mem_reg *mem = &bo->mem;
 400
 401         if (bo->mem.bus.addr) {
 402                 map->bo_kmap_type = ttm_bo_map_premapped;
 403                 map->virtual = (void *)(((u8 *)bo->mem.bus.addr) + offset);
 404         } else {
 405                 map->bo_kmap_type = ttm_bo_map_iomap;
 406                 if (mem->placement & TTM_PL_FLAG_WC)
 407                         map->virtual = ioremap_wc(bo->mem.bus.base + bo->mem.bus.offset + offset,
 408                                                   size);
 409                 else
 410                         map->virtual = ioremap_nocache(bo->mem.bus.base + bo->mem.bus.offset + offset,
 411                                                        size);
 412         }
 413         return (!map->virtual) ? -ENOMEM : 0;
 414 }
 415
 416 static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo,
 417                            unsigned long start_page,
 418                            unsigned long num_pages,
 419                            struct ttm_bo_kmap_obj *map)
 420 {
 421         struct ttm_mem_reg *mem = &bo->mem; pgprot_t prot;
 422         struct ttm_tt *ttm = bo->ttm;
 423         struct page *d;
 424         int i;
 425
 426         BUG_ON(!ttm);
 427         if (num_pages == 1 && (mem->placement & TTM_PL_FLAG_CACHED)) {
 428                 /*
 429                  * We're mapping a single page, and the desired
 430                  * page protection is consistent with the bo.
 431                  */
 432
 433                 map->bo_kmap_type = ttm_bo_map_kmap;
 434                 map->page = ttm_tt_get_page(ttm, start_page);
 435                 map->virtual = kmap(map->page);
 436         } else {
 437             /*
 438              * Populate the part we're mapping;
 439              */
 440                 for (i = start_page; i < start_page + num_pages; ++i) {
 441                         d = ttm_tt_get_page(ttm, i);
 442                         if (!d)
 443                                 return -ENOMEM;
 444                 }
 445
 446                 /*
 447                  * We need to use vmap to get the desired page protection
 448                  * or to make the buffer object look contiguous.
 449                  */
 450                 prot = (mem->placement & TTM_PL_FLAG_CACHED) ?
 451                         PAGE_KERNEL :
 452                         ttm_io_prot(mem->placement, PAGE_KERNEL);
 453                 map->bo_kmap_type = ttm_bo_map_vmap;
 454                 map->virtual = vmap(ttm->pages + start_page, num_pages,
 455                                     0, prot);
 456         }
 457         return (!map->virtual) ? -ENOMEM : 0;
 458 }
 459
 460 int ttm_bo_kmap(struct ttm_buffer_object *bo,
 461                 unsigned long start_page, unsigned long num_pages,
 462                 struct ttm_bo_kmap_obj *map)
 463 {
 464         unsigned long offset, size;
 465         int ret;
 466
 467         BUG_ON(!list_empty(&bo->swap));
 468         map->virtual = NULL;
 469         map->bo = bo;
 470         if (num_pages > bo->num_pages)
 471                 return -EINVAL;
 472         if (start_page > bo->num_pages)
 473                 return -EINVAL;
 474 #if 0
 475         if (num_pages > 1 && !DRM_SUSER(DRM_CURPROC))
 476                 return -EPERM;
 477 #endif
 478         ret = ttm_mem_io_reserve(bo->bdev, &bo->mem);
 479         if (ret)
 480                 return ret;
 481         if (!bo->mem.bus.is_iomem) {
 482                 return ttm_bo_kmap_ttm(bo, start_page, num_pages, map);
 483         } else {
 484                 offset = start_page << PAGE_SHIFT;
 485                 size = num_pages << PAGE_SHIFT;
 486                 return ttm_bo_ioremap(bo, offset, size, map);
 487         }
 488 }
 489 EXPORT_SYMBOL(ttm_bo_kmap);
 490
 491 void ttm_bo_kunmap(struct ttm_bo_kmap_obj *map)
 492 {
 493         if (!map->virtual)
 494                 return;
 495         switch (map->bo_kmap_type) {
 496         case ttm_bo_map_iomap:
 497                 iounmap(map->virtual);
 498                 ttm_mem_io_free(map->bo->bdev, &map->bo->mem);
 499                 break;
 500         case ttm_bo_map_vmap:
 501                 vunmap(map->virtual);
 502                 break;
 503         case ttm_bo_map_kmap:
 504                 kunmap(map->page);
 505                 break;
 506         case ttm_bo_map_premapped:
 507                 break;
 508         default:
 509                 BUG();
 510         }
 511         map->virtual = NULL;
 512         map->page = NULL;
 513 }
 514 EXPORT_SYMBOL(ttm_bo_kunmap);
 515
 516 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
 517                               void *sync_obj,
 518                               void *sync_obj_arg,
 519                               bool evict, bool no_wait_reserve,
 520                               bool no_wait_gpu,
 521                               struct ttm_mem_reg *new_mem)
 522 {
 523         struct ttm_bo_device *bdev = bo->bdev;
 524         struct ttm_bo_driver *driver = bdev->driver;
 525         struct ttm_mem_type_manager *man = &bdev->man[new_mem->mem_type];
 526         struct ttm_mem_reg *old_mem = &bo->mem;
 527         int ret;
 528         struct ttm_buffer_object *ghost_obj;
 529         void *tmp_obj = NULL;
 530
 531         spin_lock(&bo->lock);
 532         if (bo->sync_obj) {
 533                 tmp_obj = bo->sync_obj;
 534                 bo->sync_obj = NULL;
 535         }
 536         bo->sync_obj = driver->sync_obj_ref(sync_obj);
 537         bo->sync_obj_arg = sync_obj_arg;
 538         if (evict) {
 539                 ret = ttm_bo_wait(bo, false, false, false);
 540                 spin_unlock(&bo->lock);
 541                 if (tmp_obj)
 542                         driver->sync_obj_unref(&tmp_obj);
 543                 if (ret)
 544                         return ret;
 545
 546                 ttm_bo_free_old_node(bo);
 547                 if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 548                     (bo->ttm != NULL)) {
 549                         ttm_tt_unbind(bo->ttm);
 550                         ttm_tt_destroy(bo->ttm);
 551                         bo->ttm = NULL;
 552                 }
 553         } else {
 554                 /**
 555                  * This should help pipeline ordinary buffer moves.
 556                  *
 557                  * Hang old buffer memory on a new buffer object,
 558                  * and leave it to be released when the GPU
 559                  * operation has completed.
 560                  */
 561
 562                 set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
 563                 spin_unlock(&bo->lock);
 564                 if (tmp_obj)
 565                         driver->sync_obj_unref(&tmp_obj);
 566
 567                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
 568                 if (ret)
 569                         return ret;
 570
 571                 /**
 572                  * If we're not moving to fixed memory, the TTM object
 573                  * needs to stay alive. Otherwhise hang it on the ghost
 574                  * bo to be unbound and destroyed.
 575                  */
 576
 577                 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED))
 578                         ghost_obj->ttm = NULL;
 579                 else
 580                         bo->ttm = NULL;
 581
 582                 ttm_bo_unreserve(ghost_obj);
 583                 ttm_bo_unref(&ghost_obj);
 584         }
 585
 586         *old_mem = *new_mem;
 587         new_mem->mm_node = NULL;
 588
 589         return 0;
 590 }
 591 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);