drivers/gpu/drm/ttm/ttm_bo.c

   1 /**************************************************************************
   2  *
   3  * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27 /*
  28  * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
  29  */
  30 /* Notes:
  31  *
  32  * We store bo pointer in drm_mm_node struct so we know which bo own a
  33  * specific node. There is no protection on the pointer, thus to make
  34  * sure things don't go berserk you have to access this pointer while
  35  * holding the global lru lock and make sure anytime you free a node you
  36  * reset the pointer to NULL.
  37  */
  38
  39 #include "ttm/ttm_module.h"
  40 #include "ttm/ttm_bo_driver.h"
  41 #include "ttm/ttm_placement.h"
  42 #include <linux/jiffies.h>
  43 #include <linux/slab.h>
  44 #include <linux/sched.h>
  45 #include <linux/mm.h>
  46 #include <linux/file.h>
  47 #include <linux/module.h>
  48
  49 #define TTM_ASSERT_LOCKED(param)
  50 #define TTM_DEBUG(fmt, arg...)
  51 #define TTM_BO_HASH_ORDER 13
  52
  53 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo);
  54 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink);
  55 static void ttm_bo_global_kobj_release(struct kobject *kobj);
  56
  57 static struct attribute ttm_bo_count = {
  58         .name = "bo_count",
  59         .mode = S_IRUGO
  60 };
  61
  62 static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type)
  63 {
  64         int i;
  65
  66         for (i = 0; i <= TTM_PL_PRIV5; i++)
  67                 if (flags & (1 << i)) {
  68                         *mem_type = i;
  69                         return 0;
  70                 }
  71         return -EINVAL;
  72 }
  73
  74 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type)
  75 {
  76         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
  77
  78         printk(KERN_ERR TTM_PFX "    has_type: %d\n", man->has_type);
  79         printk(KERN_ERR TTM_PFX "    use_type: %d\n", man->use_type);
  80         printk(KERN_ERR TTM_PFX "    flags: 0x%08X\n", man->flags);
  81         printk(KERN_ERR TTM_PFX "    gpu_offset: 0x%08lX\n", man->gpu_offset);
  82         printk(KERN_ERR TTM_PFX "    size: %llu\n", man->size);
  83         printk(KERN_ERR TTM_PFX "    available_caching: 0x%08X\n",
  84                 man->available_caching);
  85         printk(KERN_ERR TTM_PFX "    default_caching: 0x%08X\n",
  86                 man->default_caching);
  87         if (mem_type != TTM_PL_SYSTEM) {
  88                 spin_lock(&bdev->glob->lru_lock);
  89                 drm_mm_debug_table(&man->manager, TTM_PFX);
  90                 spin_unlock(&bdev->glob->lru_lock);
  91         }
  92 }
  93
  94 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
  95                                         struct ttm_placement *placement)
  96 {
  97         int i, ret, mem_type;
  98
  99         printk(KERN_ERR TTM_PFX "No space for %p (%lu pages, %luK, %luM)\n",
 100                 bo, bo->mem.num_pages, bo->mem.size >> 10,
 101                 bo->mem.size >> 20);
 102         for (i = 0; i < placement->num_placement; i++) {
 103                 ret = ttm_mem_type_from_flags(placement->placement[i],
 104                                                 &mem_type);
 105                 if (ret)
 106                         return;
 107                 printk(KERN_ERR TTM_PFX "  placement[%d]=0x%08X (%d)\n",
 108                         i, placement->placement[i], mem_type);
 109                 ttm_mem_type_debug(bo->bdev, mem_type);
 110         }
 111 }
 112
 113 static ssize_t ttm_bo_global_show(struct kobject *kobj,
 114                                   struct attribute *attr,
 115                                   char *buffer)
 116 {
 117         struct ttm_bo_global *glob =
 118                 container_of(kobj, struct ttm_bo_global, kobj);
 119
 120         return snprintf(buffer, PAGE_SIZE, "%lu\n",
 121                         (unsigned long) atomic_read(&glob->bo_count));
 122 }
 123
 124 static struct attribute *ttm_bo_global_attrs[] = {
 125         &ttm_bo_count,
 126         NULL
 127 };
 128
 129 static const struct sysfs_ops ttm_bo_global_ops = {
 130         .show = &ttm_bo_global_show
 131 };
 132
 133 static struct kobj_type ttm_bo_glob_kobj_type  = {
 134         .release = &ttm_bo_global_kobj_release,
 135         .sysfs_ops = &ttm_bo_global_ops,
 136         .default_attrs = ttm_bo_global_attrs
 137 };
 138
 139
 140 static inline uint32_t ttm_bo_type_flags(unsigned type)
 141 {
 142         return 1 << (type);
 143 }
 144
 145 static void ttm_bo_release_list(struct kref *list_kref)
 146 {
 147         struct ttm_buffer_object *bo =
 148             container_of(list_kref, struct ttm_buffer_object, list_kref);
 149         struct ttm_bo_device *bdev = bo->bdev;
 150
 151         BUG_ON(atomic_read(&bo->list_kref.refcount));
 152         BUG_ON(atomic_read(&bo->kref.refcount));
 153         BUG_ON(atomic_read(&bo->cpu_writers));
 154         BUG_ON(bo->sync_obj != NULL);
 155         BUG_ON(bo->mem.mm_node != NULL);
 156         BUG_ON(!list_empty(&bo->lru));
 157         BUG_ON(!list_empty(&bo->ddestroy));
 158
 159         if (bo->ttm)
 160                 ttm_tt_destroy(bo->ttm);
 161         atomic_dec(&bo->glob->bo_count);
 162         if (bo->destroy)
 163                 bo->destroy(bo);
 164         else {
 165                 ttm_mem_global_free(bdev->glob->mem_glob, bo->acc_size);
 166                 kfree(bo);
 167         }
 168 }
 169
 170 int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
 171 {
 172
 173         if (interruptible) {
 174                 int ret = 0;
 175
 176                 ret = wait_event_interruptible(bo->event_queue,
 177                                                atomic_read(&bo->reserved) == 0);
 178                 if (unlikely(ret != 0))
 179                         return ret;
 180         } else {
 181                 wait_event(bo->event_queue, atomic_read(&bo->reserved) == 0);
 182         }
 183         return 0;
 184 }
 185 EXPORT_SYMBOL(ttm_bo_wait_unreserved);
 186
 187 static void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
 188 {
 189         struct ttm_bo_device *bdev = bo->bdev;
 190         struct ttm_mem_type_manager *man;
 191
 192         BUG_ON(!atomic_read(&bo->reserved));
 193
 194         if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) {
 195
 196                 BUG_ON(!list_empty(&bo->lru));
 197
 198                 man = &bdev->man[bo->mem.mem_type];
 199                 list_add_tail(&bo->lru, &man->lru);
 200                 kref_get(&bo->list_kref);
 201
 202                 if (bo->ttm != NULL) {
 203                         list_add_tail(&bo->swap, &bo->glob->swap_lru);
 204                         kref_get(&bo->list_kref);
 205                 }
 206         }
 207 }
 208
 209 /**
 210  * Call with the lru_lock held.
 211  */
 212
 213 static int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
 214 {
 215         int put_count = 0;
 216
 217         if (!list_empty(&bo->swap)) {
 218                 list_del_init(&bo->swap);
 219                 ++put_count;
 220         }
 221         if (!list_empty(&bo->lru)) {
 222                 list_del_init(&bo->lru);
 223                 ++put_count;
 224         }
 225
 226         /*
 227          * TODO: Add a driver hook to delete from
 228          * driver-specific LRU's here.
 229          */
 230
 231         return put_count;
 232 }
 233
 234 int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
 235                           bool interruptible,
 236                           bool no_wait, bool use_sequence, uint32_t sequence)
 237 {
 238         struct ttm_bo_global *glob = bo->glob;
 239         int ret;
 240
 241         while (unlikely(atomic_cmpxchg(&bo->reserved, 0, 1) != 0)) {
 242                 if (use_sequence && bo->seq_valid &&
 243                         (sequence - bo->val_seq < (1 << 31))) {
 244                         return -EAGAIN;
 245                 }
 246
 247                 if (no_wait)
 248                         return -EBUSY;
 249
 250                 spin_unlock(&glob->lru_lock);
 251                 ret = ttm_bo_wait_unreserved(bo, interruptible);
 252                 spin_lock(&glob->lru_lock);
 253
 254                 if (unlikely(ret))
 255                         return ret;
 256         }
 257
 258         if (use_sequence) {
 259                 bo->val_seq = sequence;
 260                 bo->seq_valid = true;
 261         } else {
 262                 bo->seq_valid = false;
 263         }
 264
 265         return 0;
 266 }
 267 EXPORT_SYMBOL(ttm_bo_reserve);
 268
 269 static void ttm_bo_ref_bug(struct kref *list_kref)
 270 {
 271         BUG();
 272 }
 273
 274 int ttm_bo_reserve(struct ttm_buffer_object *bo,
 275                    bool interruptible,
 276                    bool no_wait, bool use_sequence, uint32_t sequence)
 277 {
 278         struct ttm_bo_global *glob = bo->glob;
 279         int put_count = 0;
 280         int ret;
 281
 282         spin_lock(&glob->lru_lock);
 283         ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence,
 284                                     sequence);
 285         if (likely(ret == 0))
 286                 put_count = ttm_bo_del_from_lru(bo);
 287         spin_unlock(&glob->lru_lock);
 288
 289         while (put_count--)
 290                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 291
 292         return ret;
 293 }
 294
 295 void ttm_bo_unreserve(struct ttm_buffer_object *bo)
 296 {
 297         struct ttm_bo_global *glob = bo->glob;
 298
 299         spin_lock(&glob->lru_lock);
 300         ttm_bo_add_to_lru(bo);
 301         atomic_set(&bo->reserved, 0);
 302         wake_up_all(&bo->event_queue);
 303         spin_unlock(&glob->lru_lock);
 304 }
 305 EXPORT_SYMBOL(ttm_bo_unreserve);
 306
 307 /*
 308  * Call bo->mutex locked.
 309  */
 310 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc)
 311 {
 312         struct ttm_bo_device *bdev = bo->bdev;
 313         struct ttm_bo_global *glob = bo->glob;
 314         int ret = 0;
 315         uint32_t page_flags = 0;
 316
 317         TTM_ASSERT_LOCKED(&bo->mutex);
 318         bo->ttm = NULL;
 319
 320         if (bdev->need_dma32)
 321                 page_flags |= TTM_PAGE_FLAG_DMA32;
 322
 323         switch (bo->type) {
 324         case ttm_bo_type_device:
 325                 if (zero_alloc)
 326                         page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC;
 327         case ttm_bo_type_kernel:
 328                 bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
 329                                         page_flags, glob->dummy_read_page);
 330                 if (unlikely(bo->ttm == NULL))
 331                         ret = -ENOMEM;
 332                 break;
 333         case ttm_bo_type_user:
 334                 bo->ttm = ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT,
 335                                         page_flags | TTM_PAGE_FLAG_USER,
 336                                         glob->dummy_read_page);
 337                 if (unlikely(bo->ttm == NULL)) {
 338                         ret = -ENOMEM;
 339                         break;
 340                 }
 341
 342                 ret = ttm_tt_set_user(bo->ttm, current,
 343                                       bo->buffer_start, bo->num_pages);
 344                 if (unlikely(ret != 0))
 345                         ttm_tt_destroy(bo->ttm);
 346                 break;
 347         default:
 348                 printk(KERN_ERR TTM_PFX "Illegal buffer object type\n");
 349                 ret = -EINVAL;
 350                 break;
 351         }
 352
 353         return ret;
 354 }
 355
 356 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 357                                   struct ttm_mem_reg *mem,
 358                                   bool evict, bool interruptible,
 359                                   bool no_wait_reserve, bool no_wait_gpu)
 360 {
 361         struct ttm_bo_device *bdev = bo->bdev;
 362         bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem);
 363         bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem);
 364         struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type];
 365         struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type];
 366         int ret = 0;
 367
 368         if (old_is_pci || new_is_pci ||
 369             ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0))
 370                 ttm_bo_unmap_virtual(bo);
 371
 372         /*
 373          * Create and bind a ttm if required.
 374          */
 375
 376         if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && (bo->ttm == NULL)) {
 377                 ret = ttm_bo_add_ttm(bo, false);
 378                 if (ret)
 379                         goto out_err;
 380
 381                 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement);
 382                 if (ret)
 383                         goto out_err;
 384
 385                 if (mem->mem_type != TTM_PL_SYSTEM) {
 386                         ret = ttm_tt_bind(bo->ttm, mem);
 387                         if (ret)
 388                                 goto out_err;
 389                 }
 390
 391                 if (bo->mem.mem_type == TTM_PL_SYSTEM) {
 392                         bo->mem = *mem;
 393                         mem->mm_node = NULL;
 394                         goto moved;
 395                 }
 396
 397         }
 398
 399         if (bdev->driver->move_notify)
 400                 bdev->driver->move_notify(bo, mem);
 401
 402         if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
 403             !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED))
 404                 ret = ttm_bo_move_ttm(bo, evict, no_wait_reserve, no_wait_gpu, mem);
 405         else if (bdev->driver->move)
 406                 ret = bdev->driver->move(bo, evict, interruptible,
 407                                          no_wait_reserve, no_wait_gpu, mem);
 408         else
 409                 ret = ttm_bo_move_memcpy(bo, evict, no_wait_reserve, no_wait_gpu, mem);
 410
 411         if (ret)
 412                 goto out_err;
 413
 414 moved:
 415         if (bo->evicted) {
 416                 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement);
 417                 if (ret)
 418                         printk(KERN_ERR TTM_PFX "Can not flush read caches\n");
 419                 bo->evicted = false;
 420         }
 421
 422         if (bo->mem.mm_node) {
 423                 spin_lock(&bo->lock);
 424                 bo->offset = (bo->mem.mm_node->start << PAGE_SHIFT) +
 425                     bdev->man[bo->mem.mem_type].gpu_offset;
 426                 bo->cur_placement = bo->mem.placement;
 427                 spin_unlock(&bo->lock);
 428         } else
 429                 bo->offset = 0;
 430
 431         return 0;
 432
 433 out_err:
 434         new_man = &bdev->man[bo->mem.mem_type];
 435         if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) {
 436                 ttm_tt_unbind(bo->ttm);
 437                 ttm_tt_destroy(bo->ttm);
 438                 bo->ttm = NULL;
 439         }
 440
 441         return ret;
 442 }
 443
 444 /**
 445  * If bo idle, remove from delayed- and lru lists, and unref.
 446  * If not idle, and already on delayed list, do nothing.
 447  * If not idle, and not on delayed list, put on delayed list,
 448  *   up the list_kref and schedule a delayed list check.
 449  */
 450
 451 static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all)
 452 {
 453         struct ttm_bo_device *bdev = bo->bdev;
 454         struct ttm_bo_global *glob = bo->glob;
 455         struct ttm_bo_driver *driver = bdev->driver;
 456         int ret;
 457
 458         spin_lock(&bo->lock);
 459         (void) ttm_bo_wait(bo, false, false, !remove_all);
 460
 461         if (!bo->sync_obj) {
 462                 int put_count;
 463
 464                 spin_unlock(&bo->lock);
 465
 466                 spin_lock(&glob->lru_lock);
 467                 put_count = ttm_bo_del_from_lru(bo);
 468
 469                 ret = ttm_bo_reserve_locked(bo, false, false, false, 0);
 470                 BUG_ON(ret);
 471                 if (bo->ttm)
 472                         ttm_tt_unbind(bo->ttm);
 473
 474                 if (!list_empty(&bo->ddestroy)) {
 475                         list_del_init(&bo->ddestroy);
 476                         ++put_count;
 477                 }
 478                 spin_unlock(&glob->lru_lock);
 479                 ttm_bo_mem_put(bo, &bo->mem);
 480
 481                 atomic_set(&bo->reserved, 0);
 482
 483                 while (put_count--)
 484                         kref_put(&bo->list_kref, ttm_bo_ref_bug);
 485
 486                 return 0;
 487         }
 488
 489         spin_lock(&glob->lru_lock);
 490         if (list_empty(&bo->ddestroy)) {
 491                 void *sync_obj = bo->sync_obj;
 492                 void *sync_obj_arg = bo->sync_obj_arg;
 493
 494                 kref_get(&bo->list_kref);
 495                 list_add_tail(&bo->ddestroy, &bdev->ddestroy);
 496                 spin_unlock(&glob->lru_lock);
 497                 spin_unlock(&bo->lock);
 498
 499                 if (sync_obj)
 500                         driver->sync_obj_flush(sync_obj, sync_obj_arg);
 501                 schedule_delayed_work(&bdev->wq,
 502                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 503                 ret = 0;
 504
 505         } else {
 506                 spin_unlock(&glob->lru_lock);
 507                 spin_unlock(&bo->lock);
 508                 ret = -EBUSY;
 509         }
 510
 511         return ret;
 512 }
 513
 514 /**
 515  * Traverse the delayed list, and call ttm_bo_cleanup_refs on all
 516  * encountered buffers.
 517  */
 518
 519 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
 520 {
 521         struct ttm_bo_global *glob = bdev->glob;
 522         struct ttm_buffer_object *entry = NULL;
 523         int ret = 0;
 524
 525         spin_lock(&glob->lru_lock);
 526         if (list_empty(&bdev->ddestroy))
 527                 goto out_unlock;
 528
 529         entry = list_first_entry(&bdev->ddestroy,
 530                 struct ttm_buffer_object, ddestroy);
 531         kref_get(&entry->list_kref);
 532
 533         for (;;) {
 534                 struct ttm_buffer_object *nentry = NULL;
 535
 536                 if (entry->ddestroy.next != &bdev->ddestroy) {
 537                         nentry = list_first_entry(&entry->ddestroy,
 538                                 struct ttm_buffer_object, ddestroy);
 539                         kref_get(&nentry->list_kref);
 540                 }
 541
 542                 spin_unlock(&glob->lru_lock);
 543                 ret = ttm_bo_cleanup_refs(entry, remove_all);
 544                 kref_put(&entry->list_kref, ttm_bo_release_list);
 545                 entry = nentry;
 546
 547                 if (ret || !entry)
 548                         goto out;
 549
 550                 spin_lock(&glob->lru_lock);
 551                 if (list_empty(&entry->ddestroy))
 552                         break;
 553         }
 554
 555 out_unlock:
 556         spin_unlock(&glob->lru_lock);
 557 out:
 558         if (entry)
 559                 kref_put(&entry->list_kref, ttm_bo_release_list);
 560         return ret;
 561 }
 562
 563 static void ttm_bo_delayed_workqueue(struct work_struct *work)
 564 {
 565         struct ttm_bo_device *bdev =
 566             container_of(work, struct ttm_bo_device, wq.work);
 567
 568         if (ttm_bo_delayed_delete(bdev, false)) {
 569                 schedule_delayed_work(&bdev->wq,
 570                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 571         }
 572 }
 573
 574 static void ttm_bo_release(struct kref *kref)
 575 {
 576         struct ttm_buffer_object *bo =
 577             container_of(kref, struct ttm_buffer_object, kref);
 578         struct ttm_bo_device *bdev = bo->bdev;
 579
 580         if (likely(bo->vm_node != NULL)) {
 581                 rb_erase(&bo->vm_rb, &bdev->addr_space_rb);
 582                 drm_mm_put_block(bo->vm_node);
 583                 bo->vm_node = NULL;
 584         }
 585         write_unlock(&bdev->vm_lock);
 586         ttm_bo_cleanup_refs(bo, false);
 587         kref_put(&bo->list_kref, ttm_bo_release_list);
 588         write_lock(&bdev->vm_lock);
 589 }
 590
 591 void ttm_bo_unref(struct ttm_buffer_object **p_bo)
 592 {
 593         struct ttm_buffer_object *bo = *p_bo;
 594         struct ttm_bo_device *bdev = bo->bdev;
 595
 596         *p_bo = NULL;
 597         write_lock(&bdev->vm_lock);
 598         kref_put(&bo->kref, ttm_bo_release);
 599         write_unlock(&bdev->vm_lock);
 600 }
 601 EXPORT_SYMBOL(ttm_bo_unref);
 602
 603 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev)
 604 {
 605         return cancel_delayed_work_sync(&bdev->wq);
 606 }
 607 EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue);
 608
 609 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched)
 610 {
 611         if (resched)
 612                 schedule_delayed_work(&bdev->wq,
 613                                       ((HZ / 100) < 1) ? 1 : HZ / 100);
 614 }
 615 EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue);
 616
 617 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
 618                         bool no_wait_reserve, bool no_wait_gpu)
 619 {
 620         struct ttm_bo_device *bdev = bo->bdev;
 621         struct ttm_mem_reg evict_mem;
 622         struct ttm_placement placement;
 623         int ret = 0;
 624
 625         spin_lock(&bo->lock);
 626         ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
 627         spin_unlock(&bo->lock);
 628
 629         if (unlikely(ret != 0)) {
 630                 if (ret != -ERESTARTSYS) {
 631                         printk(KERN_ERR TTM_PFX
 632                                "Failed to expire sync object before "
 633                                "buffer eviction.\n");
 634                 }
 635                 goto out;
 636         }
 637
 638         BUG_ON(!atomic_read(&bo->reserved));
 639
 640         evict_mem = bo->mem;
 641         evict_mem.mm_node = NULL;
 642         evict_mem.bus.io_reserved = false;
 643
 644         placement.fpfn = 0;
 645         placement.lpfn = 0;
 646         placement.num_placement = 0;
 647         placement.num_busy_placement = 0;
 648         bdev->driver->evict_flags(bo, &placement);
 649         ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible,
 650                                 no_wait_reserve, no_wait_gpu);
 651         if (ret) {
 652                 if (ret != -ERESTARTSYS) {
 653                         printk(KERN_ERR TTM_PFX
 654                                "Failed to find memory space for "
 655                                "buffer 0x%p eviction.\n", bo);
 656                         ttm_bo_mem_space_debug(bo, &placement);
 657                 }
 658                 goto out;
 659         }
 660
 661         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
 662                                      no_wait_reserve, no_wait_gpu);
 663         if (ret) {
 664                 if (ret != -ERESTARTSYS)
 665                         printk(KERN_ERR TTM_PFX "Buffer eviction failed\n");
 666                 ttm_bo_mem_put(bo, &evict_mem);
 667                 goto out;
 668         }
 669         bo->evicted = true;
 670 out:
 671         return ret;
 672 }
 673
 674 static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 675                                 uint32_t mem_type,
 676                                 bool interruptible, bool no_wait_reserve,
 677                                 bool no_wait_gpu)
 678 {
 679         struct ttm_bo_global *glob = bdev->glob;
 680         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 681         struct ttm_buffer_object *bo;
 682         int ret, put_count = 0;
 683
 684 retry:
 685         spin_lock(&glob->lru_lock);
 686         if (list_empty(&man->lru)) {
 687                 spin_unlock(&glob->lru_lock);
 688                 return -EBUSY;
 689         }
 690
 691         bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru);
 692         kref_get(&bo->list_kref);
 693
 694         ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0);
 695
 696         if (unlikely(ret == -EBUSY)) {
 697                 spin_unlock(&glob->lru_lock);
 698                 if (likely(!no_wait_gpu))
 699                         ret = ttm_bo_wait_unreserved(bo, interruptible);
 700
 701                 kref_put(&bo->list_kref, ttm_bo_release_list);
 702
 703                 /**
 704                  * We *need* to retry after releasing the lru lock.
 705                  */
 706
 707                 if (unlikely(ret != 0))
 708                         return ret;
 709                 goto retry;
 710         }
 711
 712         put_count = ttm_bo_del_from_lru(bo);
 713         spin_unlock(&glob->lru_lock);
 714
 715         BUG_ON(ret != 0);
 716
 717         while (put_count--)
 718                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
 719
 720         ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu);
 721         ttm_bo_unreserve(bo);
 722
 723         kref_put(&bo->list_kref, ttm_bo_release_list);
 724         return ret;
 725 }
 726
 727 static int ttm_bo_man_get_node(struct ttm_buffer_object *bo,
 728                                 struct ttm_mem_type_manager *man,
 729                                 struct ttm_placement *placement,
 730                                 struct ttm_mem_reg *mem,
 731                                 struct drm_mm_node **node)
 732 {
 733         struct ttm_bo_global *glob = bo->glob;
 734         unsigned long lpfn;
 735         int ret;
 736
 737         lpfn = placement->lpfn;
 738         if (!lpfn)
 739                 lpfn = man->size;
 740         *node = NULL;
 741         do {
 742                 ret = drm_mm_pre_get(&man->manager);
 743                 if (unlikely(ret))
 744                         return ret;
 745
 746                 spin_lock(&glob->lru_lock);
 747                 *node = drm_mm_search_free_in_range(&man->manager,
 748                                         mem->num_pages, mem->page_alignment,
 749                                         placement->fpfn, lpfn, 1);
 750                 if (unlikely(*node == NULL)) {
 751                         spin_unlock(&glob->lru_lock);
 752                         return 0;
 753                 }
 754                 *node = drm_mm_get_block_atomic_range(*node, mem->num_pages,
 755                                                         mem->page_alignment,
 756                                                         placement->fpfn,
 757                                                         lpfn);
 758                 spin_unlock(&glob->lru_lock);
 759         } while (*node == NULL);
 760         return 0;
 761 }
 762
 763 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 764 {
 765         struct ttm_bo_global *glob = bo->glob;
 766
 767         if (mem->mm_node) {
 768                 spin_lock(&glob->lru_lock);
 769                 drm_mm_put_block(mem->mm_node);
 770                 spin_unlock(&glob->lru_lock);
 771                 mem->mm_node = NULL;
 772         }
 773 }
 774 EXPORT_SYMBOL(ttm_bo_mem_put);
 775
 776 /**
 777  * Repeatedly evict memory from the LRU for @mem_type until we create enough
 778  * space, or we've evicted everything and there isn't enough space.
 779  */
 780 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
 781                                         uint32_t mem_type,
 782                                         struct ttm_placement *placement,
 783                                         struct ttm_mem_reg *mem,
 784                                         bool interruptible,
 785                                         bool no_wait_reserve,
 786                                         bool no_wait_gpu)
 787 {
 788         struct ttm_bo_device *bdev = bo->bdev;
 789         struct ttm_bo_global *glob = bdev->glob;
 790         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
 791         struct drm_mm_node *node;
 792         int ret;
 793
 794         do {
 795                 ret = ttm_bo_man_get_node(bo, man, placement, mem, &node);
 796                 if (unlikely(ret != 0))
 797                         return ret;
 798                 if (node)
 799                         break;
 800                 spin_lock(&glob->lru_lock);
 801                 if (list_empty(&man->lru)) {
 802                         spin_unlock(&glob->lru_lock);
 803                         break;
 804                 }
 805                 spin_unlock(&glob->lru_lock);
 806                 ret = ttm_mem_evict_first(bdev, mem_type, interruptible,
 807                                                 no_wait_reserve, no_wait_gpu);
 808                 if (unlikely(ret != 0))
 809                         return ret;
 810         } while (1);
 811         if (node == NULL)
 812                 return -ENOMEM;
 813         mem->mm_node = node;
 814         mem->mem_type = mem_type;
 815         return 0;
 816 }
 817
 818 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
 819                                       uint32_t cur_placement,
 820                                       uint32_t proposed_placement)
 821 {
 822         uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING;
 823         uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING;
 824
 825         /**
 826          * Keep current caching if possible.
 827          */
 828
 829         if ((cur_placement & caching) != 0)
 830                 result |= (cur_placement & caching);
 831         else if ((man->default_caching & caching) != 0)
 832                 result |= man->default_caching;
 833         else if ((TTM_PL_FLAG_CACHED & caching) != 0)
 834                 result |= TTM_PL_FLAG_CACHED;
 835         else if ((TTM_PL_FLAG_WC & caching) != 0)
 836                 result |= TTM_PL_FLAG_WC;
 837         else if ((TTM_PL_FLAG_UNCACHED & caching) != 0)
 838                 result |= TTM_PL_FLAG_UNCACHED;
 839
 840         return result;
 841 }
 842
 843 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man,
 844                                  bool disallow_fixed,
 845                                  uint32_t mem_type,
 846                                  uint32_t proposed_placement,
 847                                  uint32_t *masked_placement)
 848 {
 849         uint32_t cur_flags = ttm_bo_type_flags(mem_type);
 850
 851         if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && disallow_fixed)
 852                 return false;
 853
 854         if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0)
 855                 return false;
 856
 857         if ((proposed_placement & man->available_caching) == 0)
 858                 return false;
 859
 860         cur_flags |= (proposed_placement & man->available_caching);
 861
 862         *masked_placement = cur_flags;
 863         return true;
 864 }
 865
 866 /**
 867  * Creates space for memory region @mem according to its type.
 868  *
 869  * This function first searches for free space in compatible memory types in
 870  * the priority order defined by the driver.  If free space isn't found, then
 871  * ttm_bo_mem_force_space is attempted in priority order to evict and find
 872  * space.
 873  */
 874 int ttm_bo_mem_space(struct ttm_buffer_object *bo,
 875                         struct ttm_placement *placement,
 876                         struct ttm_mem_reg *mem,
 877                         bool interruptible, bool no_wait_reserve,
 878                         bool no_wait_gpu)
 879 {
 880         struct ttm_bo_device *bdev = bo->bdev;
 881         struct ttm_mem_type_manager *man;
 882         uint32_t mem_type = TTM_PL_SYSTEM;
 883         uint32_t cur_flags = 0;
 884         bool type_found = false;
 885         bool type_ok = false;
 886         bool has_erestartsys = false;
 887         struct drm_mm_node *node = NULL;
 888         int i, ret;
 889
 890         mem->mm_node = NULL;
 891         for (i = 0; i < placement->num_placement; ++i) {
 892                 ret = ttm_mem_type_from_flags(placement->placement[i],
 893                                                 &mem_type);
 894                 if (ret)
 895                         return ret;
 896                 man = &bdev->man[mem_type];
 897
 898                 type_ok = ttm_bo_mt_compatible(man,
 899                                                 bo->type == ttm_bo_type_user,
 900                                                 mem_type,
 901                                                 placement->placement[i],
 902                                                 &cur_flags);
 903
 904                 if (!type_ok)
 905                         continue;
 906
 907                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
 908                                                   cur_flags);
 909                 /*
 910                  * Use the access and other non-mapping-related flag bits from
 911                  * the memory placement flags to the current flags
 912                  */
 913                 ttm_flag_masked(&cur_flags, placement->placement[i],
 914                                 ~TTM_PL_MASK_MEMTYPE);
 915
 916                 if (mem_type == TTM_PL_SYSTEM)
 917                         break;
 918
 919                 if (man->has_type && man->use_type) {
 920                         type_found = true;
 921                         ret = ttm_bo_man_get_node(bo, man, placement, mem,
 922                                                         &node);
 923                         if (unlikely(ret))
 924                                 return ret;
 925                 }
 926                 if (node)
 927                         break;
 928         }
 929
 930         if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || node) {
 931                 mem->mm_node = node;
 932                 mem->mem_type = mem_type;
 933                 mem->placement = cur_flags;
 934                 return 0;
 935         }
 936
 937         if (!type_found)
 938                 return -EINVAL;
 939
 940         for (i = 0; i < placement->num_busy_placement; ++i) {
 941                 ret = ttm_mem_type_from_flags(placement->busy_placement[i],
 942                                                 &mem_type);
 943                 if (ret)
 944                         return ret;
 945                 man = &bdev->man[mem_type];
 946                 if (!man->has_type)
 947                         continue;
 948                 if (!ttm_bo_mt_compatible(man,
 949                                                 bo->type == ttm_bo_type_user,
 950                                                 mem_type,
 951                                                 placement->busy_placement[i],
 952                                                 &cur_flags))
 953                         continue;
 954
 955                 cur_flags = ttm_bo_select_caching(man, bo->mem.placement,
 956                                                   cur_flags);
 957                 /*
 958                  * Use the access and other non-mapping-related flag bits from
 959                  * the memory placement flags to the current flags
 960                  */
 961                 ttm_flag_masked(&cur_flags, placement->busy_placement[i],
 962                                 ~TTM_PL_MASK_MEMTYPE);
 963
 964
 965                 if (mem_type == TTM_PL_SYSTEM) {
 966                         mem->mem_type = mem_type;
 967                         mem->placement = cur_flags;
 968                         mem->mm_node = NULL;
 969                         return 0;
 970                 }
 971
 972                 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem,
 973                                                 interruptible, no_wait_reserve, no_wait_gpu);
 974                 if (ret == 0 && mem->mm_node) {
 975                         mem->placement = cur_flags;
 976                         return 0;
 977                 }
 978                 if (ret == -ERESTARTSYS)
 979                         has_erestartsys = true;
 980         }
 981         ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM;
 982         return ret;
 983 }
 984 EXPORT_SYMBOL(ttm_bo_mem_space);
 985
 986 int ttm_bo_wait_cpu(struct ttm_buffer_object *bo, bool no_wait)
 987 {
 988         if ((atomic_read(&bo->cpu_writers) > 0) && no_wait)
 989                 return -EBUSY;
 990
 991         return wait_event_interruptible(bo->event_queue,
 992                                         atomic_read(&bo->cpu_writers) == 0);
 993 }
 994 EXPORT_SYMBOL(ttm_bo_wait_cpu);
 995
 996 int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 997                         struct ttm_placement *placement,
 998                         bool interruptible, bool no_wait_reserve,
 999                         bool no_wait_gpu)
1000 {
1001         struct ttm_bo_global *glob = bo->glob;
1002         int ret = 0;
1003         struct ttm_mem_reg mem;
1004
1005         BUG_ON(!atomic_read(&bo->reserved));
1006
1007         /*
1008          * FIXME: It's possible to pipeline buffer moves.
1009          * Have the driver move function wait for idle when necessary,
1010          * instead of doing it here.
1011          */
1012         spin_lock(&bo->lock);
1013         ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu);
1014         spin_unlock(&bo->lock);
1015         if (ret)
1016                 return ret;
1017         mem.num_pages = bo->num_pages;
1018         mem.size = mem.num_pages << PAGE_SHIFT;
1019         mem.page_alignment = bo->mem.page_alignment;
1020         mem.bus.io_reserved = false;
1021         /*
1022          * Determine where to move the buffer.
1023          */
1024         ret = ttm_bo_mem_space(bo, placement, &mem, interruptible, no_wait_reserve, no_wait_gpu);
1025         if (ret)
1026                 goto out_unlock;
1027         ret = ttm_bo_handle_move_mem(bo, &mem, false, interruptible, no_wait_reserve, no_wait_gpu);
1028 out_unlock:
1029         if (ret && mem.mm_node) {
1030                 spin_lock(&glob->lru_lock);
1031                 drm_mm_put_block(mem.mm_node);
1032                 spin_unlock(&glob->lru_lock);
1033         }
1034         return ret;
1035 }
1036
1037 static int ttm_bo_mem_compat(struct ttm_placement *placement,
1038                              struct ttm_mem_reg *mem)
1039 {
1040         int i;
1041         struct drm_mm_node *node = mem->mm_node;
1042
1043         if (node && placement->lpfn != 0 &&
1044             (node->start < placement->fpfn ||
1045              node->start + node->size > placement->lpfn))
1046                 return -1;
1047
1048         for (i = 0; i < placement->num_placement; i++) {
1049                 if ((placement->placement[i] & mem->placement &
1050                         TTM_PL_MASK_CACHING) &&
1051                         (placement->placement[i] & mem->placement &
1052                         TTM_PL_MASK_MEM))
1053                         return i;
1054         }
1055         return -1;
1056 }
1057
1058 int ttm_bo_validate(struct ttm_buffer_object *bo,
1059                         struct ttm_placement *placement,
1060                         bool interruptible, bool no_wait_reserve,
1061                         bool no_wait_gpu)
1062 {
1063         int ret;
1064
1065         BUG_ON(!atomic_read(&bo->reserved));
1066         /* Check that range is valid */
1067         if (placement->lpfn || placement->fpfn)
1068                 if (placement->fpfn > placement->lpfn ||
1069                         (placement->lpfn - placement->fpfn) < bo->num_pages)
1070                         return -EINVAL;
1071         /*
1072          * Check whether we need to move buffer.
1073          */
1074         ret = ttm_bo_mem_compat(placement, &bo->mem);
1075         if (ret < 0) {
1076                 ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu);
1077                 if (ret)
1078                         return ret;
1079         } else {
1080                 /*
1081                  * Use the access and other non-mapping-related flag bits from
1082                  * the compatible memory placement flags to the active flags
1083                  */
1084                 ttm_flag_masked(&bo->mem.placement, placement->placement[ret],
1085                                 ~TTM_PL_MASK_MEMTYPE);
1086         }
1087         /*
1088          * We might need to add a TTM.
1089          */
1090         if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
1091                 ret = ttm_bo_add_ttm(bo, true);
1092                 if (ret)
1093                         return ret;
1094         }
1095         return 0;
1096 }
1097 EXPORT_SYMBOL(ttm_bo_validate);
1098
1099 int ttm_bo_check_placement(struct ttm_buffer_object *bo,
1100                                 struct ttm_placement *placement)
1101 {
1102         int i;
1103
1104         if (placement->fpfn || placement->lpfn) {
1105                 if (bo->mem.num_pages > (placement->lpfn - placement->fpfn)) {
1106                         printk(KERN_ERR TTM_PFX "Page number range to small "
1107                                 "Need %lu pages, range is [%u, %u]\n",
1108                                 bo->mem.num_pages, placement->fpfn,
1109                                 placement->lpfn);
1110                         return -EINVAL;
1111                 }
1112         }
1113         for (i = 0; i < placement->num_placement; i++) {
1114                 if (!capable(CAP_SYS_ADMIN)) {
1115                         if (placement->placement[i] & TTM_PL_FLAG_NO_EVICT) {
1116                                 printk(KERN_ERR TTM_PFX "Need to be root to "
1117                                         "modify NO_EVICT status.\n");
1118                                 return -EINVAL;
1119                         }
1120                 }
1121         }
1122         for (i = 0; i < placement->num_busy_placement; i++) {
1123                 if (!capable(CAP_SYS_ADMIN)) {
1124                         if (placement->busy_placement[i] & TTM_PL_FLAG_NO_EVICT) {
1125                                 printk(KERN_ERR TTM_PFX "Need to be root to "
1126                                         "modify NO_EVICT status.\n");
1127                                 return -EINVAL;
1128                         }
1129                 }
1130         }
1131         return 0;
1132 }
1133
1134 int ttm_bo_init(struct ttm_bo_device *bdev,
1135                 struct ttm_buffer_object *bo,
1136                 unsigned long size,
1137                 enum ttm_bo_type type,
1138                 struct ttm_placement *placement,
1139                 uint32_t page_alignment,
1140                 unsigned long buffer_start,
1141                 bool interruptible,
1142                 struct file *persistant_swap_storage,
1143                 size_t acc_size,
1144                 void (*destroy) (struct ttm_buffer_object *))
1145 {
1146         int ret = 0;
1147         unsigned long num_pages;
1148
1149         size += buffer_start & ~PAGE_MASK;
1150         num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1151         if (num_pages == 0) {
1152                 printk(KERN_ERR TTM_PFX "Illegal buffer object size.\n");
1153                 return -EINVAL;
1154         }
1155         bo->destroy = destroy;
1156
1157         spin_lock_init(&bo->lock);
1158         kref_init(&bo->kref);
1159         kref_init(&bo->list_kref);
1160         atomic_set(&bo->cpu_writers, 0);
1161         atomic_set(&bo->reserved, 1);
1162         init_waitqueue_head(&bo->event_queue);
1163         INIT_LIST_HEAD(&bo->lru);
1164         INIT_LIST_HEAD(&bo->ddestroy);
1165         INIT_LIST_HEAD(&bo->swap);
1166         bo->bdev = bdev;
1167         bo->glob = bdev->glob;
1168         bo->type = type;
1169         bo->num_pages = num_pages;
1170         bo->mem.size = num_pages << PAGE_SHIFT;
1171         bo->mem.mem_type = TTM_PL_SYSTEM;
1172         bo->mem.num_pages = bo->num_pages;
1173         bo->mem.mm_node = NULL;
1174         bo->mem.page_alignment = page_alignment;
1175         bo->mem.bus.io_reserved = false;
1176         bo->buffer_start = buffer_start & PAGE_MASK;
1177         bo->priv_flags = 0;
1178         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
1179         bo->seq_valid = false;
1180         bo->persistant_swap_storage = persistant_swap_storage;
1181         bo->acc_size = acc_size;
1182         atomic_inc(&bo->glob->bo_count);
1183
1184         ret = ttm_bo_check_placement(bo, placement);
1185         if (unlikely(ret != 0))
1186                 goto out_err;
1187
1188         /*
1189          * For ttm_bo_type_device buffers, allocate
1190          * address space from the device.
1191          */
1192         if (bo->type == ttm_bo_type_device) {
1193                 ret = ttm_bo_setup_vm(bo);
1194                 if (ret)
1195                         goto out_err;
1196         }
1197
1198         ret = ttm_bo_validate(bo, placement, interruptible, false, false);
1199         if (ret)
1200                 goto out_err;
1201
1202         ttm_bo_unreserve(bo);
1203         return 0;
1204
1205 out_err:
1206         ttm_bo_unreserve(bo);
1207         ttm_bo_unref(&bo);
1208
1209         return ret;
1210 }
1211 EXPORT_SYMBOL(ttm_bo_init);
1212
1213 static inline size_t ttm_bo_size(struct ttm_bo_global *glob,
1214                                  unsigned long num_pages)
1215 {
1216         size_t page_array_size = (num_pages * sizeof(void *) + PAGE_SIZE - 1) &
1217             PAGE_MASK;
1218
1219         return glob->ttm_bo_size + 2 * page_array_size;
1220 }
1221
1222 int ttm_bo_create(struct ttm_bo_device *bdev,
1223                         unsigned long size,
1224                         enum ttm_bo_type type,
1225                         struct ttm_placement *placement,
1226                         uint32_t page_alignment,
1227                         unsigned long buffer_start,
1228                         bool interruptible,
1229                         struct file *persistant_swap_storage,
1230                         struct ttm_buffer_object **p_bo)
1231 {
1232         struct ttm_buffer_object *bo;
1233         struct ttm_mem_global *mem_glob = bdev->glob->mem_glob;
1234         int ret;
1235
1236         size_t acc_size =
1237             ttm_bo_size(bdev->glob, (size + PAGE_SIZE - 1) >> PAGE_SHIFT);
1238         ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false);
1239         if (unlikely(ret != 0))
1240                 return ret;
1241
1242         bo = kzalloc(sizeof(*bo), GFP_KERNEL);
1243
1244         if (unlikely(bo == NULL)) {
1245                 ttm_mem_global_free(mem_glob, acc_size);
1246                 return -ENOMEM;
1247         }
1248
1249         ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment,
1250                                 buffer_start, interruptible,
1251                                 persistant_swap_storage, acc_size, NULL);
1252         if (likely(ret == 0))
1253                 *p_bo = bo;
1254
1255         return ret;
1256 }
1257
1258 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
1259                                         unsigned mem_type, bool allow_errors)
1260 {
1261         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1262         struct ttm_bo_global *glob = bdev->glob;
1263         int ret;
1264
1265         /*
1266          * Can't use standard list traversal since we're unlocking.
1267          */
1268
1269         spin_lock(&glob->lru_lock);
1270         while (!list_empty(&man->lru)) {
1271                 spin_unlock(&glob->lru_lock);
1272                 ret = ttm_mem_evict_first(bdev, mem_type, false, false, false);
1273                 if (ret) {
1274                         if (allow_errors) {
1275                                 return ret;
1276                         } else {
1277                                 printk(KERN_ERR TTM_PFX
1278                                         "Cleanup eviction failed\n");
1279                         }
1280                 }
1281                 spin_lock(&glob->lru_lock);
1282         }
1283         spin_unlock(&glob->lru_lock);
1284         return 0;
1285 }
1286
1287 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1288 {
1289         struct ttm_bo_global *glob = bdev->glob;
1290         struct ttm_mem_type_manager *man;
1291         int ret = -EINVAL;
1292
1293         if (mem_type >= TTM_NUM_MEM_TYPES) {
1294                 printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", mem_type);
1295                 return ret;
1296         }
1297         man = &bdev->man[mem_type];
1298
1299         if (!man->has_type) {
1300                 printk(KERN_ERR TTM_PFX "Trying to take down uninitialized "
1301                        "memory manager type %u\n", mem_type);
1302                 return ret;
1303         }
1304
1305         man->use_type = false;
1306         man->has_type = false;
1307
1308         ret = 0;
1309         if (mem_type > 0) {
1310                 ttm_bo_force_list_clean(bdev, mem_type, false);
1311
1312                 spin_lock(&glob->lru_lock);
1313                 if (drm_mm_clean(&man->manager))
1314                         drm_mm_takedown(&man->manager);
1315                 else
1316                         ret = -EBUSY;
1317
1318                 spin_unlock(&glob->lru_lock);
1319         }
1320
1321         return ret;
1322 }
1323 EXPORT_SYMBOL(ttm_bo_clean_mm);
1324
1325 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type)
1326 {
1327         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
1328
1329         if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) {
1330                 printk(KERN_ERR TTM_PFX
1331                        "Illegal memory manager memory type %u.\n",
1332                        mem_type);
1333                 return -EINVAL;
1334         }
1335
1336         if (!man->has_type) {
1337                 printk(KERN_ERR TTM_PFX
1338                        "Memory type %u has not been initialized.\n",
1339                        mem_type);
1340                 return 0;
1341         }
1342
1343         return ttm_bo_force_list_clean(bdev, mem_type, true);
1344 }
1345 EXPORT_SYMBOL(ttm_bo_evict_mm);
1346
1347 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
1348                         unsigned long p_size)
1349 {
1350         int ret = -EINVAL;
1351         struct ttm_mem_type_manager *man;
1352
1353         if (type >= TTM_NUM_MEM_TYPES) {
1354                 printk(KERN_ERR TTM_PFX "Illegal memory type %d\n", type);
1355                 return ret;
1356         }
1357
1358         man = &bdev->man[type];
1359         if (man->has_type) {
1360                 printk(KERN_ERR TTM_PFX
1361                        "Memory manager already initialized for type %d\n",
1362                        type);
1363                 return ret;
1364         }
1365
1366         ret = bdev->driver->init_mem_type(bdev, type, man);
1367         if (ret)
1368                 return ret;
1369
1370         ret = 0;
1371         if (type != TTM_PL_SYSTEM) {
1372                 if (!p_size) {
1373                         printk(KERN_ERR TTM_PFX
1374                                "Zero size memory manager type %d\n",
1375                                type);
1376                         return ret;
1377                 }
1378                 ret = drm_mm_init(&man->manager, 0, p_size);
1379                 if (ret)
1380                         return ret;
1381         }
1382         man->has_type = true;
1383         man->use_type = true;
1384         man->size = p_size;
1385
1386         INIT_LIST_HEAD(&man->lru);
1387
1388         return 0;
1389 }
1390 EXPORT_SYMBOL(ttm_bo_init_mm);
1391
1392 static void ttm_bo_global_kobj_release(struct kobject *kobj)
1393 {
1394         struct ttm_bo_global *glob =
1395                 container_of(kobj, struct ttm_bo_global, kobj);
1396
1397         ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink);
1398         __free_page(glob->dummy_read_page);
1399         kfree(glob);
1400 }
1401
1402 void ttm_bo_global_release(struct drm_global_reference *ref)
1403 {
1404         struct ttm_bo_global *glob = ref->object;
1405
1406         kobject_del(&glob->kobj);
1407         kobject_put(&glob->kobj);
1408 }
1409 EXPORT_SYMBOL(ttm_bo_global_release);
1410
1411 int ttm_bo_global_init(struct drm_global_reference *ref)
1412 {
1413         struct ttm_bo_global_ref *bo_ref =
1414                 container_of(ref, struct ttm_bo_global_ref, ref);
1415         struct ttm_bo_global *glob = ref->object;
1416         int ret;
1417
1418         mutex_init(&glob->device_list_mutex);
1419         spin_lock_init(&glob->lru_lock);
1420         glob->mem_glob = bo_ref->mem_glob;
1421         glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32);
1422
1423         if (unlikely(glob->dummy_read_page == NULL)) {
1424                 ret = -ENOMEM;
1425                 goto out_no_drp;
1426         }
1427
1428         INIT_LIST_HEAD(&glob->swap_lru);
1429         INIT_LIST_HEAD(&glob->device_list);
1430
1431         ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout);
1432         ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink);
1433         if (unlikely(ret != 0)) {
1434                 printk(KERN_ERR TTM_PFX
1435                        "Could not register buffer object swapout.\n");
1436                 goto out_no_shrink;
1437         }
1438
1439         glob->ttm_bo_extra_size =
1440                 ttm_round_pot(sizeof(struct ttm_tt)) +
1441                 ttm_round_pot(sizeof(struct ttm_backend));
1442
1443         glob->ttm_bo_size = glob->ttm_bo_extra_size +
1444                 ttm_round_pot(sizeof(struct ttm_buffer_object));
1445
1446         atomic_set(&glob->bo_count, 0);
1447
1448         ret = kobject_init_and_add(
1449                 &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects");
1450         if (unlikely(ret != 0))
1451                 kobject_put(&glob->kobj);
1452         return ret;
1453 out_no_shrink:
1454         __free_page(glob->dummy_read_page);
1455 out_no_drp:
1456         kfree(glob);
1457         return ret;
1458 }
1459 EXPORT_SYMBOL(ttm_bo_global_init);
1460
1461
1462 int ttm_bo_device_release(struct ttm_bo_device *bdev)
1463 {
1464         int ret = 0;
1465         unsigned i = TTM_NUM_MEM_TYPES;
1466         struct ttm_mem_type_manager *man;
1467         struct ttm_bo_global *glob = bdev->glob;
1468
1469         while (i--) {
1470                 man = &bdev->man[i];
1471                 if (man->has_type) {
1472                         man->use_type = false;
1473                         if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) {
1474                                 ret = -EBUSY;
1475                                 printk(KERN_ERR TTM_PFX
1476                                        "DRM memory manager type %d "
1477                                        "is not clean.\n", i);
1478                         }
1479                         man->has_type = false;
1480                 }
1481         }
1482
1483         mutex_lock(&glob->device_list_mutex);
1484         list_del(&bdev->device_list);
1485         mutex_unlock(&glob->device_list_mutex);
1486
1487         if (!cancel_delayed_work(&bdev->wq))
1488                 flush_scheduled_work();
1489
1490         while (ttm_bo_delayed_delete(bdev, true))
1491                 ;
1492
1493         spin_lock(&glob->lru_lock);
1494         if (list_empty(&bdev->ddestroy))
1495                 TTM_DEBUG("Delayed destroy list was clean\n");
1496
1497         if (list_empty(&bdev->man[0].lru))
1498                 TTM_DEBUG("Swap list was clean\n");
1499         spin_unlock(&glob->lru_lock);
1500
1501         BUG_ON(!drm_mm_clean(&bdev->addr_space_mm));
1502         write_lock(&bdev->vm_lock);
1503         drm_mm_takedown(&bdev->addr_space_mm);
1504         write_unlock(&bdev->vm_lock);
1505
1506         return ret;
1507 }
1508 EXPORT_SYMBOL(ttm_bo_device_release);
1509
1510 int ttm_bo_device_init(struct ttm_bo_device *bdev,
1511                        struct ttm_bo_global *glob,
1512                        struct ttm_bo_driver *driver,
1513                        uint64_t file_page_offset,
1514                        bool need_dma32)
1515 {
1516         int ret = -EINVAL;
1517
1518         rwlock_init(&bdev->vm_lock);
1519         bdev->driver = driver;
1520
1521         memset(bdev->man, 0, sizeof(bdev->man));
1522
1523         /*
1524          * Initialize the system memory buffer type.
1525          * Other types need to be driver / IOCTL initialized.
1526          */
1527         ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0);
1528         if (unlikely(ret != 0))
1529                 goto out_no_sys;
1530
1531         bdev->addr_space_rb = RB_ROOT;
1532         ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000);
1533         if (unlikely(ret != 0))
1534                 goto out_no_addr_mm;
1535
1536         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
1537         bdev->nice_mode = true;
1538         INIT_LIST_HEAD(&bdev->ddestroy);
1539         bdev->dev_mapping = NULL;
1540         bdev->glob = glob;
1541         bdev->need_dma32 = need_dma32;
1542
1543         mutex_lock(&glob->device_list_mutex);
1544         list_add_tail(&bdev->device_list, &glob->device_list);
1545         mutex_unlock(&glob->device_list_mutex);
1546
1547         return 0;
1548 out_no_addr_mm:
1549         ttm_bo_clean_mm(bdev, 0);
1550 out_no_sys:
1551         return ret;
1552 }
1553 EXPORT_SYMBOL(ttm_bo_device_init);
1554
1555 /*
1556  * buffer object vm functions.
1557  */
1558
1559 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
1560 {
1561         struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
1562
1563         if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) {
1564                 if (mem->mem_type == TTM_PL_SYSTEM)
1565                         return false;
1566
1567                 if (man->flags & TTM_MEMTYPE_FLAG_CMA)
1568                         return false;
1569
1570                 if (mem->placement & TTM_PL_FLAG_CACHED)
1571                         return false;
1572         }
1573         return true;
1574 }
1575
1576 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo)
1577 {
1578         struct ttm_bo_device *bdev = bo->bdev;
1579         loff_t offset = (loff_t) bo->addr_space_offset;
1580         loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT;
1581
1582         if (!bdev->dev_mapping)
1583                 return;
1584         unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1);
1585         ttm_mem_io_free(bdev, &bo->mem);
1586 }
1587 EXPORT_SYMBOL(ttm_bo_unmap_virtual);
1588
1589 static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo)
1590 {
1591         struct ttm_bo_device *bdev = bo->bdev;
1592         struct rb_node **cur = &bdev->addr_space_rb.rb_node;
1593         struct rb_node *parent = NULL;
1594         struct ttm_buffer_object *cur_bo;
1595         unsigned long offset = bo->vm_node->start;
1596         unsigned long cur_offset;
1597
1598         while (*cur) {
1599                 parent = *cur;
1600                 cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb);
1601                 cur_offset = cur_bo->vm_node->start;
1602                 if (offset < cur_offset)
1603                         cur = &parent->rb_left;
1604                 else if (offset > cur_offset)
1605                         cur = &parent->rb_right;
1606                 else
1607                         BUG();
1608         }
1609
1610         rb_link_node(&bo->vm_rb, parent, cur);
1611         rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb);
1612 }
1613
1614 /**
1615  * ttm_bo_setup_vm:
1616  *
1617  * @bo: the buffer to allocate address space for
1618  *
1619  * Allocate address space in the drm device so that applications
1620  * can mmap the buffer and access the contents. This only
1621  * applies to ttm_bo_type_device objects as others are not
1622  * placed in the drm device address space.
1623  */
1624
1625 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo)
1626 {
1627         struct ttm_bo_device *bdev = bo->bdev;
1628         int ret;
1629
1630 retry_pre_get:
1631         ret = drm_mm_pre_get(&bdev->addr_space_mm);
1632         if (unlikely(ret != 0))
1633                 return ret;
1634
1635         write_lock(&bdev->vm_lock);
1636         bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm,
1637                                          bo->mem.num_pages, 0, 0);
1638
1639         if (unlikely(bo->vm_node == NULL)) {
1640                 ret = -ENOMEM;
1641                 goto out_unlock;
1642         }
1643
1644         bo->vm_node = drm_mm_get_block_atomic(bo->vm_node,
1645                                               bo->mem.num_pages, 0);
1646
1647         if (unlikely(bo->vm_node == NULL)) {
1648                 write_unlock(&bdev->vm_lock);
1649                 goto retry_pre_get;
1650         }
1651
1652         ttm_bo_vm_insert_rb(bo);
1653         write_unlock(&bdev->vm_lock);
1654         bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT;
1655
1656         return 0;
1657 out_unlock:
1658         write_unlock(&bdev->vm_lock);
1659         return ret;
1660 }
1661
1662 int ttm_bo_wait(struct ttm_buffer_object *bo,
1663                 bool lazy, bool interruptible, bool no_wait)
1664 {
1665         struct ttm_bo_driver *driver = bo->bdev->driver;
1666         void *sync_obj;
1667         void *sync_obj_arg;
1668         int ret = 0;
1669
1670         if (likely(bo->sync_obj == NULL))
1671                 return 0;
1672
1673         while (bo->sync_obj) {
1674
1675                 if (driver->sync_obj_signaled(bo->sync_obj, bo->sync_obj_arg)) {
1676                         void *tmp_obj = bo->sync_obj;
1677                         bo->sync_obj = NULL;
1678                         clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
1679                         spin_unlock(&bo->lock);
1680                         driver->sync_obj_unref(&tmp_obj);
1681                         spin_lock(&bo->lock);
1682                         continue;
1683                 }
1684
1685                 if (no_wait)
1686                         return -EBUSY;
1687
1688                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
1689                 sync_obj_arg = bo->sync_obj_arg;
1690                 spin_unlock(&bo->lock);
1691                 ret = driver->sync_obj_wait(sync_obj, sync_obj_arg,
1692                                             lazy, interruptible);
1693                 if (unlikely(ret != 0)) {
1694                         driver->sync_obj_unref(&sync_obj);
1695                         spin_lock(&bo->lock);
1696                         return ret;
1697                 }
1698                 spin_lock(&bo->lock);
1699                 if (likely(bo->sync_obj == sync_obj &&
1700                            bo->sync_obj_arg == sync_obj_arg)) {
1701                         void *tmp_obj = bo->sync_obj;
1702                         bo->sync_obj = NULL;
1703                         clear_bit(TTM_BO_PRIV_FLAG_MOVING,
1704                                   &bo->priv_flags);
1705                         spin_unlock(&bo->lock);
1706                         driver->sync_obj_unref(&sync_obj);
1707                         driver->sync_obj_unref(&tmp_obj);
1708                         spin_lock(&bo->lock);
1709                 } else {
1710                         spin_unlock(&bo->lock);
1711                         driver->sync_obj_unref(&sync_obj);
1712                         spin_lock(&bo->lock);
1713                 }
1714         }
1715         return 0;
1716 }
1717 EXPORT_SYMBOL(ttm_bo_wait);
1718
1719 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait)
1720 {
1721         int ret = 0;
1722
1723         /*
1724          * Using ttm_bo_reserve makes sure the lru lists are updated.
1725          */
1726
1727         ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
1728         if (unlikely(ret != 0))
1729                 return ret;
1730         spin_lock(&bo->lock);
1731         ret = ttm_bo_wait(bo, false, true, no_wait);
1732         spin_unlock(&bo->lock);
1733         if (likely(ret == 0))
1734                 atomic_inc(&bo->cpu_writers);
1735         ttm_bo_unreserve(bo);
1736         return ret;
1737 }
1738 EXPORT_SYMBOL(ttm_bo_synccpu_write_grab);
1739
1740 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo)
1741 {
1742         if (atomic_dec_and_test(&bo->cpu_writers))
1743                 wake_up_all(&bo->event_queue);
1744 }
1745 EXPORT_SYMBOL(ttm_bo_synccpu_write_release);
1746
1747 /**
1748  * A buffer object shrink method that tries to swap out the first
1749  * buffer object on the bo_global::swap_lru list.
1750  */
1751
1752 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
1753 {
1754         struct ttm_bo_global *glob =
1755             container_of(shrink, struct ttm_bo_global, shrink);
1756         struct ttm_buffer_object *bo;
1757         int ret = -EBUSY;
1758         int put_count;
1759         uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM);
1760
1761         spin_lock(&glob->lru_lock);
1762         while (ret == -EBUSY) {
1763                 if (unlikely(list_empty(&glob->swap_lru))) {
1764                         spin_unlock(&glob->lru_lock);
1765                         return -EBUSY;
1766                 }
1767
1768                 bo = list_first_entry(&glob->swap_lru,
1769                                       struct ttm_buffer_object, swap);
1770                 kref_get(&bo->list_kref);
1771
1772                 /**
1773                  * Reserve buffer. Since we unlock while sleeping, we need
1774                  * to re-check that nobody removed us from the swap-list while
1775                  * we slept.
1776                  */
1777
1778                 ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
1779                 if (unlikely(ret == -EBUSY)) {
1780                         spin_unlock(&glob->lru_lock);
1781                         ttm_bo_wait_unreserved(bo, false);
1782                         kref_put(&bo->list_kref, ttm_bo_release_list);
1783                         spin_lock(&glob->lru_lock);
1784                 }
1785         }
1786
1787         BUG_ON(ret != 0);
1788         put_count = ttm_bo_del_from_lru(bo);
1789         spin_unlock(&glob->lru_lock);
1790
1791         while (put_count--)
1792                 kref_put(&bo->list_kref, ttm_bo_ref_bug);
1793
1794         /**
1795          * Wait for GPU, then move to system cached.
1796          */
1797
1798         spin_lock(&bo->lock);
1799         ret = ttm_bo_wait(bo, false, false, false);
1800         spin_unlock(&bo->lock);
1801
1802         if (unlikely(ret != 0))
1803                 goto out;
1804
1805         if ((bo->mem.placement & swap_placement) != swap_placement) {
1806                 struct ttm_mem_reg evict_mem;
1807
1808                 evict_mem = bo->mem;
1809                 evict_mem.mm_node = NULL;
1810                 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED;
1811                 evict_mem.mem_type = TTM_PL_SYSTEM;
1812
1813                 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true,
1814                                              false, false, false);
1815                 if (unlikely(ret != 0))
1816                         goto out;
1817         }
1818
1819         ttm_bo_unmap_virtual(bo);
1820
1821         /**
1822          * Swap out. Buffer will be swapped in again as soon as
1823          * anyone tries to access a ttm page.
1824          */
1825
1826         if (bo->bdev->driver->swap_notify)
1827                 bo->bdev->driver->swap_notify(bo);
1828
1829         ret = ttm_tt_swapout(bo->ttm, bo->persistant_swap_storage);
1830 out:
1831
1832         /**
1833          *
1834          * Unreserve without putting on LRU to avoid swapping out an
1835          * already swapped buffer.
1836          */
1837
1838         atomic_set(&bo->reserved, 0);
1839         wake_up_all(&bo->event_queue);
1840         kref_put(&bo->list_kref, ttm_bo_release_list);
1841         return ret;
1842 }
1843
1844 void ttm_bo_swapout_all(struct ttm_bo_device *bdev)
1845 {
1846         while (ttm_bo_swapout(&bdev->glob->shrink) == 0)
1847                 ;
1848 }
1849 EXPORT_SYMBOL(ttm_bo_swapout_all);