drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv);
  40 static uint32_t i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv);
  41
  42 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
  43                                                   bool pipelined);
  44 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  45 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  46 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  47                                              int write);
  48 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  49                                                      uint64_t offset,
  50                                                      uint64_t size);
  51 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  52 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  53                                           bool interruptible);
  54 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  55                                        unsigned alignment,
  56                                        bool mappable,
  57                                        bool need_fence);
  58 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  59 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  60                                 struct drm_i915_gem_pwrite *args,
  61                                 struct drm_file *file_priv);
  62 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  63
  64 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  65                                     int nr_to_scan,
  66                                     gfp_t gfp_mask);
  67
  68
  69 /* some bookkeeping */
  70 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  71                                   size_t size)
  72 {
  73         dev_priv->mm.object_count++;
  74         dev_priv->mm.object_memory += size;
  75 }
  76
  77 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  78                                      size_t size)
  79 {
  80         dev_priv->mm.object_count--;
  81         dev_priv->mm.object_memory -= size;
  82 }
  83
  84 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
  85                                   struct drm_i915_gem_object *obj)
  86 {
  87         dev_priv->mm.gtt_count++;
  88         dev_priv->mm.gtt_memory += obj->gtt_space->size;
  89         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
  90                 dev_priv->mm.mappable_gtt_used +=
  91                         min_t(size_t, obj->gtt_space->size,
  92                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
  93         }
  94 }
  95
  96 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
  97                                      struct drm_i915_gem_object *obj)
  98 {
  99         dev_priv->mm.gtt_count--;
 100         dev_priv->mm.gtt_memory -= obj->gtt_space->size;
 101         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
 102                 dev_priv->mm.mappable_gtt_used -=
 103                         min_t(size_t, obj->gtt_space->size,
 104                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
 105         }
 106 }
 107
 108 /**
 109  * Update the mappable working set counters. Call _only_ when there is a change
 110  * in one of (pin|fault)_mappable and update *_mappable _before_ calling.
 111  * @mappable: new state the changed mappable flag (either pin_ or fault_).
 112  */
 113 static void
 114 i915_gem_info_update_mappable(struct drm_i915_private *dev_priv,
 115                               struct drm_i915_gem_object *obj,
 116                               bool mappable)
 117 {
 118         if (mappable) {
 119                 if (obj->pin_mappable && obj->fault_mappable)
 120                         /* Combined state was already mappable. */
 121                         return;
 122                 dev_priv->mm.gtt_mappable_count++;
 123                 dev_priv->mm.gtt_mappable_memory += obj->gtt_space->size;
 124         } else {
 125                 if (obj->pin_mappable || obj->fault_mappable)
 126                         /* Combined state still mappable. */
 127                         return;
 128                 dev_priv->mm.gtt_mappable_count--;
 129                 dev_priv->mm.gtt_mappable_memory -= obj->gtt_space->size;
 130         }
 131 }
 132
 133 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
 134                                   struct drm_i915_gem_object *obj,
 135                                   bool mappable)
 136 {
 137         dev_priv->mm.pin_count++;
 138         dev_priv->mm.pin_memory += obj->gtt_space->size;
 139         if (mappable) {
 140                 obj->pin_mappable = true;
 141                 i915_gem_info_update_mappable(dev_priv, obj, true);
 142         }
 143 }
 144
 145 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
 146                                      struct drm_i915_gem_object *obj)
 147 {
 148         dev_priv->mm.pin_count--;
 149         dev_priv->mm.pin_memory -= obj->gtt_space->size;
 150         if (obj->pin_mappable) {
 151                 obj->pin_mappable = false;
 152                 i915_gem_info_update_mappable(dev_priv, obj, false);
 153         }
 154 }
 155
 156 int
 157 i915_gem_check_is_wedged(struct drm_device *dev)
 158 {
 159         struct drm_i915_private *dev_priv = dev->dev_private;
 160         struct completion *x = &dev_priv->error_completion;
 161         unsigned long flags;
 162         int ret;
 163
 164         if (!atomic_read(&dev_priv->mm.wedged))
 165                 return 0;
 166
 167         ret = wait_for_completion_interruptible(x);
 168         if (ret)
 169                 return ret;
 170
 171         /* Success, we reset the GPU! */
 172         if (!atomic_read(&dev_priv->mm.wedged))
 173                 return 0;
 174
 175         /* GPU is hung, bump the completion count to account for
 176          * the token we just consumed so that we never hit zero and
 177          * end up waiting upon a subsequent completion event that
 178          * will never happen.
 179          */
 180         spin_lock_irqsave(&x->wait.lock, flags);
 181         x->done++;
 182         spin_unlock_irqrestore(&x->wait.lock, flags);
 183         return -EIO;
 184 }
 185
 186 static int i915_mutex_lock_interruptible(struct drm_device *dev)
 187 {
 188         struct drm_i915_private *dev_priv = dev->dev_private;
 189         int ret;
 190
 191         ret = i915_gem_check_is_wedged(dev);
 192         if (ret)
 193                 return ret;
 194
 195         ret = mutex_lock_interruptible(&dev->struct_mutex);
 196         if (ret)
 197                 return ret;
 198
 199         if (atomic_read(&dev_priv->mm.wedged)) {
 200                 mutex_unlock(&dev->struct_mutex);
 201                 return -EAGAIN;
 202         }
 203
 204         WARN_ON(i915_verify_lists(dev));
 205         return 0;
 206 }
 207
 208 static inline bool
 209 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
 210 {
 211         return obj_priv->gtt_space &&
 212                 !obj_priv->active &&
 213                 obj_priv->pin_count == 0;
 214 }
 215
 216 int i915_gem_do_init(struct drm_device *dev,
 217                      unsigned long start,
 218                      unsigned long mappable_end,
 219                      unsigned long end)
 220 {
 221         drm_i915_private_t *dev_priv = dev->dev_private;
 222
 223         if (start >= end ||
 224             (start & (PAGE_SIZE - 1)) != 0 ||
 225             (end & (PAGE_SIZE - 1)) != 0) {
 226                 return -EINVAL;
 227         }
 228
 229         drm_mm_init(&dev_priv->mm.gtt_space, start,
 230                     end - start);
 231
 232         dev_priv->mm.gtt_total = end - start;
 233         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 234         dev_priv->mm.gtt_mappable_end = mappable_end;
 235
 236         return 0;
 237 }
 238
 239 int
 240 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 241                     struct drm_file *file_priv)
 242 {
 243         struct drm_i915_gem_init *args = data;
 244         int ret;
 245
 246         mutex_lock(&dev->struct_mutex);
 247         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 248         mutex_unlock(&dev->struct_mutex);
 249
 250         return ret;
 251 }
 252
 253 int
 254 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 255                             struct drm_file *file_priv)
 256 {
 257         struct drm_i915_private *dev_priv = dev->dev_private;
 258         struct drm_i915_gem_get_aperture *args = data;
 259
 260         if (!(dev->driver->driver_features & DRIVER_GEM))
 261                 return -ENODEV;
 262
 263         mutex_lock(&dev->struct_mutex);
 264         args->aper_size = dev_priv->mm.gtt_total;
 265         args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
 266         mutex_unlock(&dev->struct_mutex);
 267
 268         return 0;
 269 }
 270
 271
 272 /**
 273  * Creates a new mm object and returns a handle to it.
 274  */
 275 int
 276 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 277                       struct drm_file *file_priv)
 278 {
 279         struct drm_i915_gem_create *args = data;
 280         struct drm_gem_object *obj;
 281         int ret;
 282         u32 handle;
 283
 284         args->size = roundup(args->size, PAGE_SIZE);
 285
 286         /* Allocate the new object */
 287         obj = i915_gem_alloc_object(dev, args->size);
 288         if (obj == NULL)
 289                 return -ENOMEM;
 290
 291         ret = drm_gem_handle_create(file_priv, obj, &handle);
 292         if (ret) {
 293                 drm_gem_object_release(obj);
 294                 i915_gem_info_remove_obj(dev->dev_private, obj->size);
 295                 kfree(obj);
 296                 return ret;
 297         }
 298
 299         /* drop reference from allocate - handle holds it now */
 300         drm_gem_object_unreference(obj);
 301         trace_i915_gem_object_create(obj);
 302
 303         args->handle = handle;
 304         return 0;
 305 }
 306
 307 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 308 {
 309         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 310         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 311
 312         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 313                 obj_priv->tiling_mode != I915_TILING_NONE;
 314 }
 315
 316 static inline void
 317 slow_shmem_copy(struct page *dst_page,
 318                 int dst_offset,
 319                 struct page *src_page,
 320                 int src_offset,
 321                 int length)
 322 {
 323         char *dst_vaddr, *src_vaddr;
 324
 325         dst_vaddr = kmap(dst_page);
 326         src_vaddr = kmap(src_page);
 327
 328         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 329
 330         kunmap(src_page);
 331         kunmap(dst_page);
 332 }
 333
 334 static inline void
 335 slow_shmem_bit17_copy(struct page *gpu_page,
 336                       int gpu_offset,
 337                       struct page *cpu_page,
 338                       int cpu_offset,
 339                       int length,
 340                       int is_read)
 341 {
 342         char *gpu_vaddr, *cpu_vaddr;
 343
 344         /* Use the unswizzled path if this page isn't affected. */
 345         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 346                 if (is_read)
 347                         return slow_shmem_copy(cpu_page, cpu_offset,
 348                                                gpu_page, gpu_offset, length);
 349                 else
 350                         return slow_shmem_copy(gpu_page, gpu_offset,
 351                                                cpu_page, cpu_offset, length);
 352         }
 353
 354         gpu_vaddr = kmap(gpu_page);
 355         cpu_vaddr = kmap(cpu_page);
 356
 357         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 358          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 359          */
 360         while (length > 0) {
 361                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 362                 int this_length = min(cacheline_end - gpu_offset, length);
 363                 int swizzled_gpu_offset = gpu_offset ^ 64;
 364
 365                 if (is_read) {
 366                         memcpy(cpu_vaddr + cpu_offset,
 367                                gpu_vaddr + swizzled_gpu_offset,
 368                                this_length);
 369                 } else {
 370                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 371                                cpu_vaddr + cpu_offset,
 372                                this_length);
 373                 }
 374                 cpu_offset += this_length;
 375                 gpu_offset += this_length;
 376                 length -= this_length;
 377         }
 378
 379         kunmap(cpu_page);
 380         kunmap(gpu_page);
 381 }
 382
 383 /**
 384  * This is the fast shmem pread path, which attempts to copy_from_user directly
 385  * from the backing pages of the object to the user's address space.  On a
 386  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 387  */
 388 static int
 389 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 390                           struct drm_i915_gem_pread *args,
 391                           struct drm_file *file_priv)
 392 {
 393         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 394         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 395         ssize_t remain;
 396         loff_t offset;
 397         char __user *user_data;
 398         int page_offset, page_length;
 399
 400         user_data = (char __user *) (uintptr_t) args->data_ptr;
 401         remain = args->size;
 402
 403         obj_priv = to_intel_bo(obj);
 404         offset = args->offset;
 405
 406         while (remain > 0) {
 407                 struct page *page;
 408                 char *vaddr;
 409                 int ret;
 410
 411                 /* Operation in this page
 412                  *
 413                  * page_offset = offset within page
 414                  * page_length = bytes to copy for this page
 415                  */
 416                 page_offset = offset & (PAGE_SIZE-1);
 417                 page_length = remain;
 418                 if ((page_offset + remain) > PAGE_SIZE)
 419                         page_length = PAGE_SIZE - page_offset;
 420
 421                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 422                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 423                 if (IS_ERR(page))
 424                         return PTR_ERR(page);
 425
 426                 vaddr = kmap_atomic(page);
 427                 ret = __copy_to_user_inatomic(user_data,
 428                                               vaddr + page_offset,
 429                                               page_length);
 430                 kunmap_atomic(vaddr);
 431
 432                 mark_page_accessed(page);
 433                 page_cache_release(page);
 434                 if (ret)
 435                         return -EFAULT;
 436
 437                 remain -= page_length;
 438                 user_data += page_length;
 439                 offset += page_length;
 440         }
 441
 442         return 0;
 443 }
 444
 445 /**
 446  * This is the fallback shmem pread path, which allocates temporary storage
 447  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 448  * can copy out of the object's backing pages while holding the struct mutex
 449  * and not take page faults.
 450  */
 451 static int
 452 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 453                           struct drm_i915_gem_pread *args,
 454                           struct drm_file *file_priv)
 455 {
 456         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 457         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 458         struct mm_struct *mm = current->mm;
 459         struct page **user_pages;
 460         ssize_t remain;
 461         loff_t offset, pinned_pages, i;
 462         loff_t first_data_page, last_data_page, num_pages;
 463         int shmem_page_offset;
 464         int data_page_index, data_page_offset;
 465         int page_length;
 466         int ret;
 467         uint64_t data_ptr = args->data_ptr;
 468         int do_bit17_swizzling;
 469
 470         remain = args->size;
 471
 472         /* Pin the user pages containing the data.  We can't fault while
 473          * holding the struct mutex, yet we want to hold it while
 474          * dereferencing the user data.
 475          */
 476         first_data_page = data_ptr / PAGE_SIZE;
 477         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 478         num_pages = last_data_page - first_data_page + 1;
 479
 480         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 481         if (user_pages == NULL)
 482                 return -ENOMEM;
 483
 484         mutex_unlock(&dev->struct_mutex);
 485         down_read(&mm->mmap_sem);
 486         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 487                                       num_pages, 1, 0, user_pages, NULL);
 488         up_read(&mm->mmap_sem);
 489         mutex_lock(&dev->struct_mutex);
 490         if (pinned_pages < num_pages) {
 491                 ret = -EFAULT;
 492                 goto out;
 493         }
 494
 495         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 496                                                         args->offset,
 497                                                         args->size);
 498         if (ret)
 499                 goto out;
 500
 501         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 502
 503         obj_priv = to_intel_bo(obj);
 504         offset = args->offset;
 505
 506         while (remain > 0) {
 507                 struct page *page;
 508
 509                 /* Operation in this page
 510                  *
 511                  * shmem_page_offset = offset within page in shmem file
 512                  * data_page_index = page number in get_user_pages return
 513                  * data_page_offset = offset with data_page_index page.
 514                  * page_length = bytes to copy for this page
 515                  */
 516                 shmem_page_offset = offset & ~PAGE_MASK;
 517                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 518                 data_page_offset = data_ptr & ~PAGE_MASK;
 519
 520                 page_length = remain;
 521                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 522                         page_length = PAGE_SIZE - shmem_page_offset;
 523                 if ((data_page_offset + page_length) > PAGE_SIZE)
 524                         page_length = PAGE_SIZE - data_page_offset;
 525
 526                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 527                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 528                 if (IS_ERR(page))
 529                         return PTR_ERR(page);
 530
 531                 if (do_bit17_swizzling) {
 532                         slow_shmem_bit17_copy(page,
 533                                               shmem_page_offset,
 534                                               user_pages[data_page_index],
 535                                               data_page_offset,
 536                                               page_length,
 537                                               1);
 538                 } else {
 539                         slow_shmem_copy(user_pages[data_page_index],
 540                                         data_page_offset,
 541                                         page,
 542                                         shmem_page_offset,
 543                                         page_length);
 544                 }
 545
 546                 mark_page_accessed(page);
 547                 page_cache_release(page);
 548
 549                 remain -= page_length;
 550                 data_ptr += page_length;
 551                 offset += page_length;
 552         }
 553
 554 out:
 555         for (i = 0; i < pinned_pages; i++) {
 556                 SetPageDirty(user_pages[i]);
 557                 mark_page_accessed(user_pages[i]);
 558                 page_cache_release(user_pages[i]);
 559         }
 560         drm_free_large(user_pages);
 561
 562         return ret;
 563 }
 564
 565 /**
 566  * Reads data from the object referenced by handle.
 567  *
 568  * On error, the contents of *data are undefined.
 569  */
 570 int
 571 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 572                      struct drm_file *file_priv)
 573 {
 574         struct drm_i915_gem_pread *args = data;
 575         struct drm_gem_object *obj;
 576         struct drm_i915_gem_object *obj_priv;
 577         int ret = 0;
 578
 579         ret = i915_mutex_lock_interruptible(dev);
 580         if (ret)
 581                 return ret;
 582
 583         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 584         if (obj == NULL) {
 585                 ret = -ENOENT;
 586                 goto unlock;
 587         }
 588         obj_priv = to_intel_bo(obj);
 589
 590         /* Bounds check source.  */
 591         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 592                 ret = -EINVAL;
 593                 goto out;
 594         }
 595
 596         if (args->size == 0)
 597                 goto out;
 598
 599         if (!access_ok(VERIFY_WRITE,
 600                        (char __user *)(uintptr_t)args->data_ptr,
 601                        args->size)) {
 602                 ret = -EFAULT;
 603                 goto out;
 604         }
 605
 606         ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
 607                                        args->size);
 608         if (ret) {
 609                 ret = -EFAULT;
 610                 goto out;
 611         }
 612
 613         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 614                                                         args->offset,
 615                                                         args->size);
 616         if (ret)
 617                 goto out;
 618
 619         ret = -EFAULT;
 620         if (!i915_gem_object_needs_bit17_swizzle(obj))
 621                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 622         if (ret == -EFAULT)
 623                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 624
 625 out:
 626         drm_gem_object_unreference(obj);
 627 unlock:
 628         mutex_unlock(&dev->struct_mutex);
 629         return ret;
 630 }
 631
 632 /* This is the fast write path which cannot handle
 633  * page faults in the source data
 634  */
 635
 636 static inline int
 637 fast_user_write(struct io_mapping *mapping,
 638                 loff_t page_base, int page_offset,
 639                 char __user *user_data,
 640                 int length)
 641 {
 642         char *vaddr_atomic;
 643         unsigned long unwritten;
 644
 645         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 646         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 647                                                       user_data, length);
 648         io_mapping_unmap_atomic(vaddr_atomic);
 649         return unwritten;
 650 }
 651
 652 /* Here's the write path which can sleep for
 653  * page faults
 654  */
 655
 656 static inline void
 657 slow_kernel_write(struct io_mapping *mapping,
 658                   loff_t gtt_base, int gtt_offset,
 659                   struct page *user_page, int user_offset,
 660                   int length)
 661 {
 662         char __iomem *dst_vaddr;
 663         char *src_vaddr;
 664
 665         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 666         src_vaddr = kmap(user_page);
 667
 668         memcpy_toio(dst_vaddr + gtt_offset,
 669                     src_vaddr + user_offset,
 670                     length);
 671
 672         kunmap(user_page);
 673         io_mapping_unmap(dst_vaddr);
 674 }
 675
 676 /**
 677  * This is the fast pwrite path, where we copy the data directly from the
 678  * user into the GTT, uncached.
 679  */
 680 static int
 681 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 682                          struct drm_i915_gem_pwrite *args,
 683                          struct drm_file *file_priv)
 684 {
 685         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 686         drm_i915_private_t *dev_priv = dev->dev_private;
 687         ssize_t remain;
 688         loff_t offset, page_base;
 689         char __user *user_data;
 690         int page_offset, page_length;
 691
 692         user_data = (char __user *) (uintptr_t) args->data_ptr;
 693         remain = args->size;
 694
 695         obj_priv = to_intel_bo(obj);
 696         offset = obj_priv->gtt_offset + args->offset;
 697
 698         while (remain > 0) {
 699                 /* Operation in this page
 700                  *
 701                  * page_base = page offset within aperture
 702                  * page_offset = offset within page
 703                  * page_length = bytes to copy for this page
 704                  */
 705                 page_base = (offset & ~(PAGE_SIZE-1));
 706                 page_offset = offset & (PAGE_SIZE-1);
 707                 page_length = remain;
 708                 if ((page_offset + remain) > PAGE_SIZE)
 709                         page_length = PAGE_SIZE - page_offset;
 710
 711                 /* If we get a fault while copying data, then (presumably) our
 712                  * source page isn't available.  Return the error and we'll
 713                  * retry in the slow path.
 714                  */
 715                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 716                                     page_offset, user_data, page_length))
 717
 718                         return -EFAULT;
 719
 720                 remain -= page_length;
 721                 user_data += page_length;
 722                 offset += page_length;
 723         }
 724
 725         return 0;
 726 }
 727
 728 /**
 729  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 730  * the memory and maps it using kmap_atomic for copying.
 731  *
 732  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 733  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 734  */
 735 static int
 736 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 737                          struct drm_i915_gem_pwrite *args,
 738                          struct drm_file *file_priv)
 739 {
 740         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 741         drm_i915_private_t *dev_priv = dev->dev_private;
 742         ssize_t remain;
 743         loff_t gtt_page_base, offset;
 744         loff_t first_data_page, last_data_page, num_pages;
 745         loff_t pinned_pages, i;
 746         struct page **user_pages;
 747         struct mm_struct *mm = current->mm;
 748         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 749         int ret;
 750         uint64_t data_ptr = args->data_ptr;
 751
 752         remain = args->size;
 753
 754         /* Pin the user pages containing the data.  We can't fault while
 755          * holding the struct mutex, and all of the pwrite implementations
 756          * want to hold it while dereferencing the user data.
 757          */
 758         first_data_page = data_ptr / PAGE_SIZE;
 759         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 760         num_pages = last_data_page - first_data_page + 1;
 761
 762         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 763         if (user_pages == NULL)
 764                 return -ENOMEM;
 765
 766         mutex_unlock(&dev->struct_mutex);
 767         down_read(&mm->mmap_sem);
 768         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 769                                       num_pages, 0, 0, user_pages, NULL);
 770         up_read(&mm->mmap_sem);
 771         mutex_lock(&dev->struct_mutex);
 772         if (pinned_pages < num_pages) {
 773                 ret = -EFAULT;
 774                 goto out_unpin_pages;
 775         }
 776
 777         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 778         if (ret)
 779                 goto out_unpin_pages;
 780
 781         obj_priv = to_intel_bo(obj);
 782         offset = obj_priv->gtt_offset + args->offset;
 783
 784         while (remain > 0) {
 785                 /* Operation in this page
 786                  *
 787                  * gtt_page_base = page offset within aperture
 788                  * gtt_page_offset = offset within page in aperture
 789                  * data_page_index = page number in get_user_pages return
 790                  * data_page_offset = offset with data_page_index page.
 791                  * page_length = bytes to copy for this page
 792                  */
 793                 gtt_page_base = offset & PAGE_MASK;
 794                 gtt_page_offset = offset & ~PAGE_MASK;
 795                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 796                 data_page_offset = data_ptr & ~PAGE_MASK;
 797
 798                 page_length = remain;
 799                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 800                         page_length = PAGE_SIZE - gtt_page_offset;
 801                 if ((data_page_offset + page_length) > PAGE_SIZE)
 802                         page_length = PAGE_SIZE - data_page_offset;
 803
 804                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 805                                   gtt_page_base, gtt_page_offset,
 806                                   user_pages[data_page_index],
 807                                   data_page_offset,
 808                                   page_length);
 809
 810                 remain -= page_length;
 811                 offset += page_length;
 812                 data_ptr += page_length;
 813         }
 814
 815 out_unpin_pages:
 816         for (i = 0; i < pinned_pages; i++)
 817                 page_cache_release(user_pages[i]);
 818         drm_free_large(user_pages);
 819
 820         return ret;
 821 }
 822
 823 /**
 824  * This is the fast shmem pwrite path, which attempts to directly
 825  * copy_from_user into the kmapped pages backing the object.
 826  */
 827 static int
 828 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 829                            struct drm_i915_gem_pwrite *args,
 830                            struct drm_file *file_priv)
 831 {
 832         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 833         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 834         ssize_t remain;
 835         loff_t offset;
 836         char __user *user_data;
 837         int page_offset, page_length;
 838
 839         user_data = (char __user *) (uintptr_t) args->data_ptr;
 840         remain = args->size;
 841
 842         obj_priv = to_intel_bo(obj);
 843         offset = args->offset;
 844         obj_priv->dirty = 1;
 845
 846         while (remain > 0) {
 847                 struct page *page;
 848                 char *vaddr;
 849                 int ret;
 850
 851                 /* Operation in this page
 852                  *
 853                  * page_offset = offset within page
 854                  * page_length = bytes to copy for this page
 855                  */
 856                 page_offset = offset & (PAGE_SIZE-1);
 857                 page_length = remain;
 858                 if ((page_offset + remain) > PAGE_SIZE)
 859                         page_length = PAGE_SIZE - page_offset;
 860
 861                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 862                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 863                 if (IS_ERR(page))
 864                         return PTR_ERR(page);
 865
 866                 vaddr = kmap_atomic(page, KM_USER0);
 867                 ret = __copy_from_user_inatomic(vaddr + page_offset,
 868                                                 user_data,
 869                                                 page_length);
 870                 kunmap_atomic(vaddr, KM_USER0);
 871
 872                 set_page_dirty(page);
 873                 mark_page_accessed(page);
 874                 page_cache_release(page);
 875
 876                 /* If we get a fault while copying data, then (presumably) our
 877                  * source page isn't available.  Return the error and we'll
 878                  * retry in the slow path.
 879                  */
 880                 if (ret)
 881                         return -EFAULT;
 882
 883                 remain -= page_length;
 884                 user_data += page_length;
 885                 offset += page_length;
 886         }
 887
 888         return 0;
 889 }
 890
 891 /**
 892  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 893  * the memory and maps it using kmap_atomic for copying.
 894  *
 895  * This avoids taking mmap_sem for faulting on the user's address while the
 896  * struct_mutex is held.
 897  */
 898 static int
 899 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 900                            struct drm_i915_gem_pwrite *args,
 901                            struct drm_file *file_priv)
 902 {
 903         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 904         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 905         struct mm_struct *mm = current->mm;
 906         struct page **user_pages;
 907         ssize_t remain;
 908         loff_t offset, pinned_pages, i;
 909         loff_t first_data_page, last_data_page, num_pages;
 910         int shmem_page_offset;
 911         int data_page_index,  data_page_offset;
 912         int page_length;
 913         int ret;
 914         uint64_t data_ptr = args->data_ptr;
 915         int do_bit17_swizzling;
 916
 917         remain = args->size;
 918
 919         /* Pin the user pages containing the data.  We can't fault while
 920          * holding the struct mutex, and all of the pwrite implementations
 921          * want to hold it while dereferencing the user data.
 922          */
 923         first_data_page = data_ptr / PAGE_SIZE;
 924         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 925         num_pages = last_data_page - first_data_page + 1;
 926
 927         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 928         if (user_pages == NULL)
 929                 return -ENOMEM;
 930
 931         mutex_unlock(&dev->struct_mutex);
 932         down_read(&mm->mmap_sem);
 933         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 934                                       num_pages, 0, 0, user_pages, NULL);
 935         up_read(&mm->mmap_sem);
 936         mutex_lock(&dev->struct_mutex);
 937         if (pinned_pages < num_pages) {
 938                 ret = -EFAULT;
 939                 goto out;
 940         }
 941
 942         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 943         if (ret)
 944                 goto out;
 945
 946         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 947
 948         obj_priv = to_intel_bo(obj);
 949         offset = args->offset;
 950         obj_priv->dirty = 1;
 951
 952         while (remain > 0) {
 953                 struct page *page;
 954
 955                 /* Operation in this page
 956                  *
 957                  * shmem_page_offset = offset within page in shmem file
 958                  * data_page_index = page number in get_user_pages return
 959                  * data_page_offset = offset with data_page_index page.
 960                  * page_length = bytes to copy for this page
 961                  */
 962                 shmem_page_offset = offset & ~PAGE_MASK;
 963                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 964                 data_page_offset = data_ptr & ~PAGE_MASK;
 965
 966                 page_length = remain;
 967                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 968                         page_length = PAGE_SIZE - shmem_page_offset;
 969                 if ((data_page_offset + page_length) > PAGE_SIZE)
 970                         page_length = PAGE_SIZE - data_page_offset;
 971
 972                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 973                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 974                 if (IS_ERR(page)) {
 975                         ret = PTR_ERR(page);
 976                         goto out;
 977                 }
 978
 979                 if (do_bit17_swizzling) {
 980                         slow_shmem_bit17_copy(page,
 981                                               shmem_page_offset,
 982                                               user_pages[data_page_index],
 983                                               data_page_offset,
 984                                               page_length,
 985                                               0);
 986                 } else {
 987                         slow_shmem_copy(page,
 988                                         shmem_page_offset,
 989                                         user_pages[data_page_index],
 990                                         data_page_offset,
 991                                         page_length);
 992                 }
 993
 994                 set_page_dirty(page);
 995                 mark_page_accessed(page);
 996                 page_cache_release(page);
 997
 998                 remain -= page_length;
 999                 data_ptr += page_length;
1000                 offset += page_length;
1001         }
1002
1003 out:
1004         for (i = 0; i < pinned_pages; i++)
1005                 page_cache_release(user_pages[i]);
1006         drm_free_large(user_pages);
1007
1008         return ret;
1009 }
1010
1011 /**
1012  * Writes data to the object referenced by handle.
1013  *
1014  * On error, the contents of the buffer that were to be modified are undefined.
1015  */
1016 int
1017 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1018                       struct drm_file *file)
1019 {
1020         struct drm_i915_gem_pwrite *args = data;
1021         struct drm_gem_object *obj;
1022         struct drm_i915_gem_object *obj_priv;
1023         int ret = 0;
1024
1025         ret = i915_mutex_lock_interruptible(dev);
1026         if (ret)
1027                 return ret;
1028
1029         obj = drm_gem_object_lookup(dev, file, args->handle);
1030         if (obj == NULL) {
1031                 ret = -ENOENT;
1032                 goto unlock;
1033         }
1034         obj_priv = to_intel_bo(obj);
1035
1036
1037         /* Bounds check destination. */
1038         if (args->offset > obj->size || args->size > obj->size - args->offset) {
1039                 ret = -EINVAL;
1040                 goto out;
1041         }
1042
1043         if (args->size == 0)
1044                 goto out;
1045
1046         if (!access_ok(VERIFY_READ,
1047                        (char __user *)(uintptr_t)args->data_ptr,
1048                        args->size)) {
1049                 ret = -EFAULT;
1050                 goto out;
1051         }
1052
1053         ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1054                                       args->size);
1055         if (ret) {
1056                 ret = -EFAULT;
1057                 goto out;
1058         }
1059
1060         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1061          * it would end up going through the fenced access, and we'll get
1062          * different detiling behavior between reading and writing.
1063          * pread/pwrite currently are reading and writing from the CPU
1064          * perspective, requiring manual detiling by the client.
1065          */
1066         if (obj_priv->phys_obj)
1067                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1068         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1069                  obj_priv->gtt_space &&
1070                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
1071                 ret = i915_gem_object_pin(obj, 0, true, false);
1072                 if (ret)
1073                         goto out;
1074
1075                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1076                 if (ret)
1077                         goto out_unpin;
1078
1079                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1080                 if (ret == -EFAULT)
1081                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1082
1083 out_unpin:
1084                 i915_gem_object_unpin(obj);
1085         } else {
1086                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1087                 if (ret)
1088                         goto out;
1089
1090                 ret = -EFAULT;
1091                 if (!i915_gem_object_needs_bit17_swizzle(obj))
1092                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1093                 if (ret == -EFAULT)
1094                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1095         }
1096
1097 out:
1098         drm_gem_object_unreference(obj);
1099 unlock:
1100         mutex_unlock(&dev->struct_mutex);
1101         return ret;
1102 }
1103
1104 /**
1105  * Called when user space prepares to use an object with the CPU, either
1106  * through the mmap ioctl's mapping or a GTT mapping.
1107  */
1108 int
1109 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1110                           struct drm_file *file_priv)
1111 {
1112         struct drm_i915_private *dev_priv = dev->dev_private;
1113         struct drm_i915_gem_set_domain *args = data;
1114         struct drm_gem_object *obj;
1115         struct drm_i915_gem_object *obj_priv;
1116         uint32_t read_domains = args->read_domains;
1117         uint32_t write_domain = args->write_domain;
1118         int ret;
1119
1120         if (!(dev->driver->driver_features & DRIVER_GEM))
1121                 return -ENODEV;
1122
1123         /* Only handle setting domains to types used by the CPU. */
1124         if (write_domain & I915_GEM_GPU_DOMAINS)
1125                 return -EINVAL;
1126
1127         if (read_domains & I915_GEM_GPU_DOMAINS)
1128                 return -EINVAL;
1129
1130         /* Having something in the write domain implies it's in the read
1131          * domain, and only that read domain.  Enforce that in the request.
1132          */
1133         if (write_domain != 0 && read_domains != write_domain)
1134                 return -EINVAL;
1135
1136         ret = i915_mutex_lock_interruptible(dev);
1137         if (ret)
1138                 return ret;
1139
1140         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1141         if (obj == NULL) {
1142                 ret = -ENOENT;
1143                 goto unlock;
1144         }
1145         obj_priv = to_intel_bo(obj);
1146
1147         intel_mark_busy(dev, obj);
1148
1149         if (read_domains & I915_GEM_DOMAIN_GTT) {
1150                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1151
1152                 /* Update the LRU on the fence for the CPU access that's
1153                  * about to occur.
1154                  */
1155                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1156                         struct drm_i915_fence_reg *reg =
1157                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1158                         list_move_tail(&reg->lru_list,
1159                                        &dev_priv->mm.fence_list);
1160                 }
1161
1162                 /* Silently promote "you're not bound, there was nothing to do"
1163                  * to success, since the client was just asking us to
1164                  * make sure everything was done.
1165                  */
1166                 if (ret == -EINVAL)
1167                         ret = 0;
1168         } else {
1169                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1170         }
1171
1172         /* Maintain LRU order of "inactive" objects */
1173         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1174                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1175
1176         drm_gem_object_unreference(obj);
1177 unlock:
1178         mutex_unlock(&dev->struct_mutex);
1179         return ret;
1180 }
1181
1182 /**
1183  * Called when user space has done writes to this buffer
1184  */
1185 int
1186 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1187                       struct drm_file *file_priv)
1188 {
1189         struct drm_i915_gem_sw_finish *args = data;
1190         struct drm_gem_object *obj;
1191         int ret = 0;
1192
1193         if (!(dev->driver->driver_features & DRIVER_GEM))
1194                 return -ENODEV;
1195
1196         ret = i915_mutex_lock_interruptible(dev);
1197         if (ret)
1198                 return ret;
1199
1200         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1201         if (obj == NULL) {
1202                 ret = -ENOENT;
1203                 goto unlock;
1204         }
1205
1206         /* Pinned buffers may be scanout, so flush the cache */
1207         if (to_intel_bo(obj)->pin_count)
1208                 i915_gem_object_flush_cpu_write_domain(obj);
1209
1210         drm_gem_object_unreference(obj);
1211 unlock:
1212         mutex_unlock(&dev->struct_mutex);
1213         return ret;
1214 }
1215
1216 /**
1217  * Maps the contents of an object, returning the address it is mapped
1218  * into.
1219  *
1220  * While the mapping holds a reference on the contents of the object, it doesn't
1221  * imply a ref on the object itself.
1222  */
1223 int
1224 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1225                    struct drm_file *file_priv)
1226 {
1227         struct drm_i915_private *dev_priv = dev->dev_private;
1228         struct drm_i915_gem_mmap *args = data;
1229         struct drm_gem_object *obj;
1230         loff_t offset;
1231         unsigned long addr;
1232
1233         if (!(dev->driver->driver_features & DRIVER_GEM))
1234                 return -ENODEV;
1235
1236         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1237         if (obj == NULL)
1238                 return -ENOENT;
1239
1240         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1241                 drm_gem_object_unreference_unlocked(obj);
1242                 return -E2BIG;
1243         }
1244
1245         offset = args->offset;
1246
1247         down_write(&current->mm->mmap_sem);
1248         addr = do_mmap(obj->filp, 0, args->size,
1249                        PROT_READ | PROT_WRITE, MAP_SHARED,
1250                        args->offset);
1251         up_write(&current->mm->mmap_sem);
1252         drm_gem_object_unreference_unlocked(obj);
1253         if (IS_ERR((void *)addr))
1254                 return addr;
1255
1256         args->addr_ptr = (uint64_t) addr;
1257
1258         return 0;
1259 }
1260
1261 /**
1262  * i915_gem_fault - fault a page into the GTT
1263  * vma: VMA in question
1264  * vmf: fault info
1265  *
1266  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1267  * from userspace.  The fault handler takes care of binding the object to
1268  * the GTT (if needed), allocating and programming a fence register (again,
1269  * only if needed based on whether the old reg is still valid or the object
1270  * is tiled) and inserting a new PTE into the faulting process.
1271  *
1272  * Note that the faulting process may involve evicting existing objects
1273  * from the GTT and/or fence registers to make room.  So performance may
1274  * suffer if the GTT working set is large or there are few fence registers
1275  * left.
1276  */
1277 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1278 {
1279         struct drm_gem_object *obj = vma->vm_private_data;
1280         struct drm_device *dev = obj->dev;
1281         drm_i915_private_t *dev_priv = dev->dev_private;
1282         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1283         pgoff_t page_offset;
1284         unsigned long pfn;
1285         int ret = 0;
1286         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1287
1288         /* We don't use vmf->pgoff since that has the fake offset */
1289         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1290                 PAGE_SHIFT;
1291
1292         /* Now bind it into the GTT if needed */
1293         mutex_lock(&dev->struct_mutex);
1294         BUG_ON(obj_priv->pin_count && !obj_priv->pin_mappable);
1295
1296         if (obj_priv->gtt_space) {
1297                 if (!obj_priv->mappable ||
1298                     (obj_priv->tiling_mode && !obj_priv->fenceable)) {
1299                         ret = i915_gem_object_unbind(obj);
1300                         if (ret)
1301                                 goto unlock;
1302                 }
1303         }
1304
1305         if (!obj_priv->gtt_space) {
1306                 ret = i915_gem_object_bind_to_gtt(obj, 0,
1307                                                   true, obj_priv->tiling_mode);
1308                 if (ret)
1309                         goto unlock;
1310         }
1311
1312         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1313         if (ret)
1314                 goto unlock;
1315
1316         if (!obj_priv->fault_mappable) {
1317                 obj_priv->fault_mappable = true;
1318                 i915_gem_info_update_mappable(dev_priv, obj_priv, true);
1319         }
1320
1321         /* Need a new fence register? */
1322         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1323                 ret = i915_gem_object_get_fence_reg(obj, true);
1324                 if (ret)
1325                         goto unlock;
1326         }
1327
1328         if (i915_gem_object_is_inactive(obj_priv))
1329                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1330
1331         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1332                 page_offset;
1333
1334         /* Finally, remap it using the new GTT offset */
1335         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1336 unlock:
1337         mutex_unlock(&dev->struct_mutex);
1338
1339         switch (ret) {
1340         case 0:
1341         case -ERESTARTSYS:
1342                 return VM_FAULT_NOPAGE;
1343         case -ENOMEM:
1344         case -EAGAIN:
1345                 return VM_FAULT_OOM;
1346         default:
1347                 return VM_FAULT_SIGBUS;
1348         }
1349 }
1350
1351 /**
1352  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1353  * @obj: obj in question
1354  *
1355  * GEM memory mapping works by handing back to userspace a fake mmap offset
1356  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1357  * up the object based on the offset and sets up the various memory mapping
1358  * structures.
1359  *
1360  * This routine allocates and attaches a fake offset for @obj.
1361  */
1362 static int
1363 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1364 {
1365         struct drm_device *dev = obj->dev;
1366         struct drm_gem_mm *mm = dev->mm_private;
1367         struct drm_map_list *list;
1368         struct drm_local_map *map;
1369         int ret = 0;
1370
1371         /* Set the object up for mmap'ing */
1372         list = &obj->map_list;
1373         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1374         if (!list->map)
1375                 return -ENOMEM;
1376
1377         map = list->map;
1378         map->type = _DRM_GEM;
1379         map->size = obj->size;
1380         map->handle = obj;
1381
1382         /* Get a DRM GEM mmap offset allocated... */
1383         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1384                                                     obj->size / PAGE_SIZE, 0, 0);
1385         if (!list->file_offset_node) {
1386                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1387                 ret = -ENOSPC;
1388                 goto out_free_list;
1389         }
1390
1391         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1392                                                   obj->size / PAGE_SIZE, 0);
1393         if (!list->file_offset_node) {
1394                 ret = -ENOMEM;
1395                 goto out_free_list;
1396         }
1397
1398         list->hash.key = list->file_offset_node->start;
1399         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1400         if (ret) {
1401                 DRM_ERROR("failed to add to map hash\n");
1402                 goto out_free_mm;
1403         }
1404
1405         return 0;
1406
1407 out_free_mm:
1408         drm_mm_put_block(list->file_offset_node);
1409 out_free_list:
1410         kfree(list->map);
1411         list->map = NULL;
1412
1413         return ret;
1414 }
1415
1416 /**
1417  * i915_gem_release_mmap - remove physical page mappings
1418  * @obj: obj in question
1419  *
1420  * Preserve the reservation of the mmapping with the DRM core code, but
1421  * relinquish ownership of the pages back to the system.
1422  *
1423  * It is vital that we remove the page mapping if we have mapped a tiled
1424  * object through the GTT and then lose the fence register due to
1425  * resource pressure. Similarly if the object has been moved out of the
1426  * aperture, than pages mapped into userspace must be revoked. Removing the
1427  * mapping will then trigger a page fault on the next user access, allowing
1428  * fixup by i915_gem_fault().
1429  */
1430 void
1431 i915_gem_release_mmap(struct drm_gem_object *obj)
1432 {
1433         struct drm_device *dev = obj->dev;
1434         struct drm_i915_private *dev_priv = dev->dev_private;
1435         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1436
1437         if (unlikely(obj->map_list.map && dev->dev_mapping))
1438                 unmap_mapping_range(dev->dev_mapping,
1439                                     (loff_t)obj->map_list.hash.key<<PAGE_SHIFT,
1440                                     obj->size, 1);
1441
1442         if (obj_priv->fault_mappable) {
1443                 obj_priv->fault_mappable = false;
1444                 i915_gem_info_update_mappable(dev_priv, obj_priv, false);
1445         }
1446 }
1447
1448 static void
1449 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1450 {
1451         struct drm_device *dev = obj->dev;
1452         struct drm_gem_mm *mm = dev->mm_private;
1453         struct drm_map_list *list = &obj->map_list;
1454
1455         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1456         drm_mm_put_block(list->file_offset_node);
1457         kfree(list->map);
1458         list->map = NULL;
1459 }
1460
1461 /**
1462  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1463  * @obj: object to check
1464  *
1465  * Return the required GTT alignment for an object, taking into account
1466  * potential fence register mapping if needed.
1467  */
1468 static uint32_t
1469 i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv)
1470 {
1471         struct drm_device *dev = obj_priv->base.dev;
1472
1473         /*
1474          * Minimum alignment is 4k (GTT page size), but might be greater
1475          * if a fence register is needed for the object.
1476          */
1477         if (INTEL_INFO(dev)->gen >= 4 ||
1478             obj_priv->tiling_mode == I915_TILING_NONE)
1479                 return 4096;
1480
1481         /*
1482          * Previous chips need to be aligned to the size of the smallest
1483          * fence register that can contain the object.
1484          */
1485         return i915_gem_get_gtt_size(obj_priv);
1486 }
1487
1488 static uint32_t
1489 i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv)
1490 {
1491         struct drm_device *dev = obj_priv->base.dev;
1492         uint32_t size;
1493
1494         /*
1495          * Minimum alignment is 4k (GTT page size), but might be greater
1496          * if a fence register is needed for the object.
1497          */
1498         if (INTEL_INFO(dev)->gen >= 4)
1499                 return obj_priv->base.size;
1500
1501         /*
1502          * Previous chips need to be aligned to the size of the smallest
1503          * fence register that can contain the object.
1504          */
1505         if (INTEL_INFO(dev)->gen == 3)
1506                 size = 1024*1024;
1507         else
1508                 size = 512*1024;
1509
1510         while (size < obj_priv->base.size)
1511                 size <<= 1;
1512
1513         return size;
1514 }
1515
1516 /**
1517  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1518  * @dev: DRM device
1519  * @data: GTT mapping ioctl data
1520  * @file_priv: GEM object info
1521  *
1522  * Simply returns the fake offset to userspace so it can mmap it.
1523  * The mmap call will end up in drm_gem_mmap(), which will set things
1524  * up so we can get faults in the handler above.
1525  *
1526  * The fault handler will take care of binding the object into the GTT
1527  * (since it may have been evicted to make room for something), allocating
1528  * a fence register, and mapping the appropriate aperture address into
1529  * userspace.
1530  */
1531 int
1532 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1533                         struct drm_file *file_priv)
1534 {
1535         struct drm_i915_private *dev_priv = dev->dev_private;
1536         struct drm_i915_gem_mmap_gtt *args = data;
1537         struct drm_gem_object *obj;
1538         struct drm_i915_gem_object *obj_priv;
1539         int ret;
1540
1541         if (!(dev->driver->driver_features & DRIVER_GEM))
1542                 return -ENODEV;
1543
1544         ret = i915_mutex_lock_interruptible(dev);
1545         if (ret)
1546                 return ret;
1547
1548         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1549         if (obj == NULL) {
1550                 ret = -ENOENT;
1551                 goto unlock;
1552         }
1553         obj_priv = to_intel_bo(obj);
1554
1555         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1556                 ret = -E2BIG;
1557                 goto unlock;
1558         }
1559
1560         if (obj_priv->madv != I915_MADV_WILLNEED) {
1561                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1562                 ret = -EINVAL;
1563                 goto out;
1564         }
1565
1566         if (!obj->map_list.map) {
1567                 ret = i915_gem_create_mmap_offset(obj);
1568                 if (ret)
1569                         goto out;
1570         }
1571
1572         args->offset = (u64)obj->map_list.hash.key << PAGE_SHIFT;
1573
1574 out:
1575         drm_gem_object_unreference(obj);
1576 unlock:
1577         mutex_unlock(&dev->struct_mutex);
1578         return ret;
1579 }
1580
1581 static int
1582 i915_gem_object_get_pages_gtt(struct drm_gem_object *obj,
1583                               gfp_t gfpmask)
1584 {
1585         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1586         int page_count, i;
1587         struct address_space *mapping;
1588         struct inode *inode;
1589         struct page *page;
1590
1591         /* Get the list of pages out of our struct file.  They'll be pinned
1592          * at this point until we release them.
1593          */
1594         page_count = obj->size / PAGE_SIZE;
1595         BUG_ON(obj_priv->pages != NULL);
1596         obj_priv->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1597         if (obj_priv->pages == NULL)
1598                 return -ENOMEM;
1599
1600         inode = obj->filp->f_path.dentry->d_inode;
1601         mapping = inode->i_mapping;
1602         for (i = 0; i < page_count; i++) {
1603                 page = read_cache_page_gfp(mapping, i,
1604                                            GFP_HIGHUSER |
1605                                            __GFP_COLD |
1606                                            __GFP_RECLAIMABLE |
1607                                            gfpmask);
1608                 if (IS_ERR(page))
1609                         goto err_pages;
1610
1611                 obj_priv->pages[i] = page;
1612         }
1613
1614         if (obj_priv->tiling_mode != I915_TILING_NONE)
1615                 i915_gem_object_do_bit_17_swizzle(obj);
1616
1617         return 0;
1618
1619 err_pages:
1620         while (i--)
1621                 page_cache_release(obj_priv->pages[i]);
1622
1623         drm_free_large(obj_priv->pages);
1624         obj_priv->pages = NULL;
1625         return PTR_ERR(page);
1626 }
1627
1628 static void
1629 i915_gem_object_put_pages_gtt(struct drm_gem_object *obj)
1630 {
1631         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1632         int page_count = obj->size / PAGE_SIZE;
1633         int i;
1634
1635         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1636
1637         if (obj_priv->tiling_mode != I915_TILING_NONE)
1638                 i915_gem_object_save_bit_17_swizzle(obj);
1639
1640         if (obj_priv->madv == I915_MADV_DONTNEED)
1641                 obj_priv->dirty = 0;
1642
1643         for (i = 0; i < page_count; i++) {
1644                 if (obj_priv->dirty)
1645                         set_page_dirty(obj_priv->pages[i]);
1646
1647                 if (obj_priv->madv == I915_MADV_WILLNEED)
1648                         mark_page_accessed(obj_priv->pages[i]);
1649
1650                 page_cache_release(obj_priv->pages[i]);
1651         }
1652         obj_priv->dirty = 0;
1653
1654         drm_free_large(obj_priv->pages);
1655         obj_priv->pages = NULL;
1656 }
1657
1658 static uint32_t
1659 i915_gem_next_request_seqno(struct drm_device *dev,
1660                             struct intel_ring_buffer *ring)
1661 {
1662         drm_i915_private_t *dev_priv = dev->dev_private;
1663
1664         ring->outstanding_lazy_request = true;
1665         return dev_priv->next_seqno;
1666 }
1667
1668 static void
1669 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1670                                struct intel_ring_buffer *ring)
1671 {
1672         struct drm_device *dev = obj->dev;
1673         struct drm_i915_private *dev_priv = dev->dev_private;
1674         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1675         uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1676
1677         BUG_ON(ring == NULL);
1678         obj_priv->ring = ring;
1679
1680         /* Add a reference if we're newly entering the active list. */
1681         if (!obj_priv->active) {
1682                 drm_gem_object_reference(obj);
1683                 obj_priv->active = 1;
1684         }
1685
1686         /* Move from whatever list we were on to the tail of execution. */
1687         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1688         list_move_tail(&obj_priv->ring_list, &ring->active_list);
1689         obj_priv->last_rendering_seqno = seqno;
1690 }
1691
1692 static void
1693 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1694 {
1695         struct drm_device *dev = obj->dev;
1696         drm_i915_private_t *dev_priv = dev->dev_private;
1697         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1698
1699         BUG_ON(!obj_priv->active);
1700         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1701         list_del_init(&obj_priv->ring_list);
1702         obj_priv->last_rendering_seqno = 0;
1703 }
1704
1705 /* Immediately discard the backing storage */
1706 static void
1707 i915_gem_object_truncate(struct drm_gem_object *obj)
1708 {
1709         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1710         struct inode *inode;
1711
1712         /* Our goal here is to return as much of the memory as
1713          * is possible back to the system as we are called from OOM.
1714          * To do this we must instruct the shmfs to drop all of its
1715          * backing pages, *now*. Here we mirror the actions taken
1716          * when by shmem_delete_inode() to release the backing store.
1717          */
1718         inode = obj->filp->f_path.dentry->d_inode;
1719         truncate_inode_pages(inode->i_mapping, 0);
1720         if (inode->i_op->truncate_range)
1721                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1722
1723         obj_priv->madv = __I915_MADV_PURGED;
1724 }
1725
1726 static inline int
1727 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1728 {
1729         return obj_priv->madv == I915_MADV_DONTNEED;
1730 }
1731
1732 static void
1733 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1734 {
1735         struct drm_device *dev = obj->dev;
1736         drm_i915_private_t *dev_priv = dev->dev_private;
1737         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1738
1739         if (obj_priv->pin_count != 0)
1740                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1741         else
1742                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1743         list_del_init(&obj_priv->ring_list);
1744
1745         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1746
1747         obj_priv->last_rendering_seqno = 0;
1748         obj_priv->ring = NULL;
1749         if (obj_priv->active) {
1750                 obj_priv->active = 0;
1751                 drm_gem_object_unreference(obj);
1752         }
1753         WARN_ON(i915_verify_lists(dev));
1754 }
1755
1756 static void
1757 i915_gem_process_flushing_list(struct drm_device *dev,
1758                                uint32_t flush_domains,
1759                                struct intel_ring_buffer *ring)
1760 {
1761         drm_i915_private_t *dev_priv = dev->dev_private;
1762         struct drm_i915_gem_object *obj_priv, *next;
1763
1764         list_for_each_entry_safe(obj_priv, next,
1765                                  &ring->gpu_write_list,
1766                                  gpu_write_list) {
1767                 struct drm_gem_object *obj = &obj_priv->base;
1768
1769                 if (obj->write_domain & flush_domains) {
1770                         uint32_t old_write_domain = obj->write_domain;
1771
1772                         obj->write_domain = 0;
1773                         list_del_init(&obj_priv->gpu_write_list);
1774                         i915_gem_object_move_to_active(obj, ring);
1775
1776                         /* update the fence lru list */
1777                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1778                                 struct drm_i915_fence_reg *reg =
1779                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1780                                 list_move_tail(&reg->lru_list,
1781                                                 &dev_priv->mm.fence_list);
1782                         }
1783
1784                         trace_i915_gem_object_change_domain(obj,
1785                                                             obj->read_domains,
1786                                                             old_write_domain);
1787                 }
1788         }
1789 }
1790
1791 int
1792 i915_add_request(struct drm_device *dev,
1793                  struct drm_file *file,
1794                  struct drm_i915_gem_request *request,
1795                  struct intel_ring_buffer *ring)
1796 {
1797         drm_i915_private_t *dev_priv = dev->dev_private;
1798         struct drm_i915_file_private *file_priv = NULL;
1799         uint32_t seqno;
1800         int was_empty;
1801         int ret;
1802
1803         BUG_ON(request == NULL);
1804
1805         if (file != NULL)
1806                 file_priv = file->driver_priv;
1807
1808         ret = ring->add_request(ring, &seqno);
1809         if (ret)
1810             return ret;
1811
1812         ring->outstanding_lazy_request = false;
1813
1814         request->seqno = seqno;
1815         request->ring = ring;
1816         request->emitted_jiffies = jiffies;
1817         was_empty = list_empty(&ring->request_list);
1818         list_add_tail(&request->list, &ring->request_list);
1819
1820         if (file_priv) {
1821                 spin_lock(&file_priv->mm.lock);
1822                 request->file_priv = file_priv;
1823                 list_add_tail(&request->client_list,
1824                               &file_priv->mm.request_list);
1825                 spin_unlock(&file_priv->mm.lock);
1826         }
1827
1828         if (!dev_priv->mm.suspended) {
1829                 mod_timer(&dev_priv->hangcheck_timer,
1830                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1831                 if (was_empty)
1832                         queue_delayed_work(dev_priv->wq,
1833                                            &dev_priv->mm.retire_work, HZ);
1834         }
1835         return 0;
1836 }
1837
1838 /**
1839  * Command execution barrier
1840  *
1841  * Ensures that all commands in the ring are finished
1842  * before signalling the CPU
1843  */
1844 static void
1845 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1846 {
1847         uint32_t flush_domains = 0;
1848
1849         /* The sampler always gets flushed on i965 (sigh) */
1850         if (INTEL_INFO(dev)->gen >= 4)
1851                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1852
1853         ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
1854 }
1855
1856 static inline void
1857 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1858 {
1859         struct drm_i915_file_private *file_priv = request->file_priv;
1860
1861         if (!file_priv)
1862                 return;
1863
1864         spin_lock(&file_priv->mm.lock);
1865         list_del(&request->client_list);
1866         request->file_priv = NULL;
1867         spin_unlock(&file_priv->mm.lock);
1868 }
1869
1870 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1871                                       struct intel_ring_buffer *ring)
1872 {
1873         while (!list_empty(&ring->request_list)) {
1874                 struct drm_i915_gem_request *request;
1875
1876                 request = list_first_entry(&ring->request_list,
1877                                            struct drm_i915_gem_request,
1878                                            list);
1879
1880                 list_del(&request->list);
1881                 i915_gem_request_remove_from_client(request);
1882                 kfree(request);
1883         }
1884
1885         while (!list_empty(&ring->active_list)) {
1886                 struct drm_i915_gem_object *obj_priv;
1887
1888                 obj_priv = list_first_entry(&ring->active_list,
1889                                             struct drm_i915_gem_object,
1890                                             ring_list);
1891
1892                 obj_priv->base.write_domain = 0;
1893                 list_del_init(&obj_priv->gpu_write_list);
1894                 i915_gem_object_move_to_inactive(&obj_priv->base);
1895         }
1896 }
1897
1898 void i915_gem_reset(struct drm_device *dev)
1899 {
1900         struct drm_i915_private *dev_priv = dev->dev_private;
1901         struct drm_i915_gem_object *obj_priv;
1902         int i;
1903
1904         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1905         i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1906         i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1907
1908         /* Remove anything from the flushing lists. The GPU cache is likely
1909          * to be lost on reset along with the data, so simply move the
1910          * lost bo to the inactive list.
1911          */
1912         while (!list_empty(&dev_priv->mm.flushing_list)) {
1913                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1914                                             struct drm_i915_gem_object,
1915                                             mm_list);
1916
1917                 obj_priv->base.write_domain = 0;
1918                 list_del_init(&obj_priv->gpu_write_list);
1919                 i915_gem_object_move_to_inactive(&obj_priv->base);
1920         }
1921
1922         /* Move everything out of the GPU domains to ensure we do any
1923          * necessary invalidation upon reuse.
1924          */
1925         list_for_each_entry(obj_priv,
1926                             &dev_priv->mm.inactive_list,
1927                             mm_list)
1928         {
1929                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1930         }
1931
1932         /* The fence registers are invalidated so clear them out */
1933         for (i = 0; i < 16; i++) {
1934                 struct drm_i915_fence_reg *reg;
1935
1936                 reg = &dev_priv->fence_regs[i];
1937                 if (!reg->obj)
1938                         continue;
1939
1940                 i915_gem_clear_fence_reg(reg->obj);
1941         }
1942 }
1943
1944 /**
1945  * This function clears the request list as sequence numbers are passed.
1946  */
1947 static void
1948 i915_gem_retire_requests_ring(struct drm_device *dev,
1949                               struct intel_ring_buffer *ring)
1950 {
1951         drm_i915_private_t *dev_priv = dev->dev_private;
1952         uint32_t seqno;
1953
1954         if (!ring->status_page.page_addr ||
1955             list_empty(&ring->request_list))
1956                 return;
1957
1958         WARN_ON(i915_verify_lists(dev));
1959
1960         seqno = ring->get_seqno(ring);
1961         while (!list_empty(&ring->request_list)) {
1962                 struct drm_i915_gem_request *request;
1963
1964                 request = list_first_entry(&ring->request_list,
1965                                            struct drm_i915_gem_request,
1966                                            list);
1967
1968                 if (!i915_seqno_passed(seqno, request->seqno))
1969                         break;
1970
1971                 trace_i915_gem_request_retire(dev, request->seqno);
1972
1973                 list_del(&request->list);
1974                 i915_gem_request_remove_from_client(request);
1975                 kfree(request);
1976         }
1977
1978         /* Move any buffers on the active list that are no longer referenced
1979          * by the ringbuffer to the flushing/inactive lists as appropriate.
1980          */
1981         while (!list_empty(&ring->active_list)) {
1982                 struct drm_gem_object *obj;
1983                 struct drm_i915_gem_object *obj_priv;
1984
1985                 obj_priv = list_first_entry(&ring->active_list,
1986                                             struct drm_i915_gem_object,
1987                                             ring_list);
1988
1989                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1990                         break;
1991
1992                 obj = &obj_priv->base;
1993                 if (obj->write_domain != 0)
1994                         i915_gem_object_move_to_flushing(obj);
1995                 else
1996                         i915_gem_object_move_to_inactive(obj);
1997         }
1998
1999         if (unlikely (dev_priv->trace_irq_seqno &&
2000                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
2001                 ring->user_irq_put(ring);
2002                 dev_priv->trace_irq_seqno = 0;
2003         }
2004
2005         WARN_ON(i915_verify_lists(dev));
2006 }
2007
2008 void
2009 i915_gem_retire_requests(struct drm_device *dev)
2010 {
2011         drm_i915_private_t *dev_priv = dev->dev_private;
2012
2013         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2014             struct drm_i915_gem_object *obj_priv, *tmp;
2015
2016             /* We must be careful that during unbind() we do not
2017              * accidentally infinitely recurse into retire requests.
2018              * Currently:
2019              *   retire -> free -> unbind -> wait -> retire_ring
2020              */
2021             list_for_each_entry_safe(obj_priv, tmp,
2022                                      &dev_priv->mm.deferred_free_list,
2023                                      mm_list)
2024                     i915_gem_free_object_tail(&obj_priv->base);
2025         }
2026
2027         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
2028         i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
2029         i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
2030 }
2031
2032 static void
2033 i915_gem_retire_work_handler(struct work_struct *work)
2034 {
2035         drm_i915_private_t *dev_priv;
2036         struct drm_device *dev;
2037
2038         dev_priv = container_of(work, drm_i915_private_t,
2039                                 mm.retire_work.work);
2040         dev = dev_priv->dev;
2041
2042         /* Come back later if the device is busy... */
2043         if (!mutex_trylock(&dev->struct_mutex)) {
2044                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2045                 return;
2046         }
2047
2048         i915_gem_retire_requests(dev);
2049
2050         if (!dev_priv->mm.suspended &&
2051                 (!list_empty(&dev_priv->render_ring.request_list) ||
2052                  !list_empty(&dev_priv->bsd_ring.request_list) ||
2053                  !list_empty(&dev_priv->blt_ring.request_list)))
2054                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2055         mutex_unlock(&dev->struct_mutex);
2056 }
2057
2058 int
2059 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
2060                      bool interruptible, struct intel_ring_buffer *ring)
2061 {
2062         drm_i915_private_t *dev_priv = dev->dev_private;
2063         u32 ier;
2064         int ret = 0;
2065
2066         BUG_ON(seqno == 0);
2067
2068         if (atomic_read(&dev_priv->mm.wedged))
2069                 return -EAGAIN;
2070
2071         if (ring->outstanding_lazy_request) {
2072                 struct drm_i915_gem_request *request;
2073
2074                 request = kzalloc(sizeof(*request), GFP_KERNEL);
2075                 if (request == NULL)
2076                         return -ENOMEM;
2077
2078                 ret = i915_add_request(dev, NULL, request, ring);
2079                 if (ret) {
2080                         kfree(request);
2081                         return ret;
2082                 }
2083
2084                 seqno = request->seqno;
2085         }
2086         BUG_ON(seqno == dev_priv->next_seqno);
2087
2088         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2089                 if (HAS_PCH_SPLIT(dev))
2090                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2091                 else
2092                         ier = I915_READ(IER);
2093                 if (!ier) {
2094                         DRM_ERROR("something (likely vbetool) disabled "
2095                                   "interrupts, re-enabling\n");
2096                         i915_driver_irq_preinstall(dev);
2097                         i915_driver_irq_postinstall(dev);
2098                 }
2099
2100                 trace_i915_gem_request_wait_begin(dev, seqno);
2101
2102                 ring->waiting_seqno = seqno;
2103                 ring->user_irq_get(ring);
2104                 if (interruptible)
2105                         ret = wait_event_interruptible(ring->irq_queue,
2106                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2107                                 || atomic_read(&dev_priv->mm.wedged));
2108                 else
2109                         wait_event(ring->irq_queue,
2110                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2111                                 || atomic_read(&dev_priv->mm.wedged));
2112
2113                 ring->user_irq_put(ring);
2114                 ring->waiting_seqno = 0;
2115
2116                 trace_i915_gem_request_wait_end(dev, seqno);
2117         }
2118         if (atomic_read(&dev_priv->mm.wedged))
2119                 ret = -EAGAIN;
2120
2121         if (ret && ret != -ERESTARTSYS)
2122                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2123                           __func__, ret, seqno, ring->get_seqno(ring),
2124                           dev_priv->next_seqno);
2125
2126         /* Directly dispatch request retiring.  While we have the work queue
2127          * to handle this, the waiter on a request often wants an associated
2128          * buffer to have made it to the inactive list, and we would need
2129          * a separate wait queue to handle that.
2130          */
2131         if (ret == 0)
2132                 i915_gem_retire_requests_ring(dev, ring);
2133
2134         return ret;
2135 }
2136
2137 /**
2138  * Waits for a sequence number to be signaled, and cleans up the
2139  * request and object lists appropriately for that event.
2140  */
2141 static int
2142 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2143                   struct intel_ring_buffer *ring)
2144 {
2145         return i915_do_wait_request(dev, seqno, 1, ring);
2146 }
2147
2148 static void
2149 i915_gem_flush_ring(struct drm_device *dev,
2150                     struct drm_file *file_priv,
2151                     struct intel_ring_buffer *ring,
2152                     uint32_t invalidate_domains,
2153                     uint32_t flush_domains)
2154 {
2155         ring->flush(ring, invalidate_domains, flush_domains);
2156         i915_gem_process_flushing_list(dev, flush_domains, ring);
2157 }
2158
2159 static void
2160 i915_gem_flush(struct drm_device *dev,
2161                struct drm_file *file_priv,
2162                uint32_t invalidate_domains,
2163                uint32_t flush_domains,
2164                uint32_t flush_rings)
2165 {
2166         drm_i915_private_t *dev_priv = dev->dev_private;
2167
2168         if (flush_domains & I915_GEM_DOMAIN_CPU)
2169                 drm_agp_chipset_flush(dev);
2170
2171         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2172                 if (flush_rings & RING_RENDER)
2173                         i915_gem_flush_ring(dev, file_priv,
2174                                             &dev_priv->render_ring,
2175                                             invalidate_domains, flush_domains);
2176                 if (flush_rings & RING_BSD)
2177                         i915_gem_flush_ring(dev, file_priv,
2178                                             &dev_priv->bsd_ring,
2179                                             invalidate_domains, flush_domains);
2180                 if (flush_rings & RING_BLT)
2181                         i915_gem_flush_ring(dev, file_priv,
2182                                             &dev_priv->blt_ring,
2183                                             invalidate_domains, flush_domains);
2184         }
2185 }
2186
2187 /**
2188  * Ensures that all rendering to the object has completed and the object is
2189  * safe to unbind from the GTT or access from the CPU.
2190  */
2191 static int
2192 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2193                                bool interruptible)
2194 {
2195         struct drm_device *dev = obj->dev;
2196         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2197         int ret;
2198
2199         /* This function only exists to support waiting for existing rendering,
2200          * not for emitting required flushes.
2201          */
2202         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2203
2204         /* If there is rendering queued on the buffer being evicted, wait for
2205          * it.
2206          */
2207         if (obj_priv->active) {
2208                 ret = i915_do_wait_request(dev,
2209                                            obj_priv->last_rendering_seqno,
2210                                            interruptible,
2211                                            obj_priv->ring);
2212                 if (ret)
2213                         return ret;
2214         }
2215
2216         return 0;
2217 }
2218
2219 /**
2220  * Unbinds an object from the GTT aperture.
2221  */
2222 int
2223 i915_gem_object_unbind(struct drm_gem_object *obj)
2224 {
2225         struct drm_device *dev = obj->dev;
2226         struct drm_i915_private *dev_priv = dev->dev_private;
2227         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2228         int ret = 0;
2229
2230         if (obj_priv->gtt_space == NULL)
2231                 return 0;
2232
2233         if (obj_priv->pin_count != 0) {
2234                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2235                 return -EINVAL;
2236         }
2237
2238         /* blow away mappings if mapped through GTT */
2239         i915_gem_release_mmap(obj);
2240
2241         /* Move the object to the CPU domain to ensure that
2242          * any possible CPU writes while it's not in the GTT
2243          * are flushed when we go to remap it. This will
2244          * also ensure that all pending GPU writes are finished
2245          * before we unbind.
2246          */
2247         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2248         if (ret == -ERESTARTSYS)
2249                 return ret;
2250         /* Continue on if we fail due to EIO, the GPU is hung so we
2251          * should be safe and we need to cleanup or else we might
2252          * cause memory corruption through use-after-free.
2253          */
2254         if (ret) {
2255                 i915_gem_clflush_object(obj);
2256                 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2257         }
2258
2259         /* release the fence reg _after_ flushing */
2260         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2261                 i915_gem_clear_fence_reg(obj);
2262
2263         drm_unbind_agp(obj_priv->agp_mem);
2264         drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2265
2266         i915_gem_object_put_pages_gtt(obj);
2267
2268         i915_gem_info_remove_gtt(dev_priv, obj_priv);
2269         list_del_init(&obj_priv->mm_list);
2270         obj_priv->fenceable = true;
2271         obj_priv->mappable = true;
2272
2273         drm_mm_put_block(obj_priv->gtt_space);
2274         obj_priv->gtt_space = NULL;
2275         obj_priv->gtt_offset = 0;
2276
2277         if (i915_gem_object_is_purgeable(obj_priv))
2278                 i915_gem_object_truncate(obj);
2279
2280         trace_i915_gem_object_unbind(obj);
2281
2282         return ret;
2283 }
2284
2285 static int i915_ring_idle(struct drm_device *dev,
2286                           struct intel_ring_buffer *ring)
2287 {
2288         if (list_empty(&ring->gpu_write_list))
2289                 return 0;
2290
2291         i915_gem_flush_ring(dev, NULL, ring,
2292                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2293         return i915_wait_request(dev,
2294                                  i915_gem_next_request_seqno(dev, ring),
2295                                  ring);
2296 }
2297
2298 int
2299 i915_gpu_idle(struct drm_device *dev)
2300 {
2301         drm_i915_private_t *dev_priv = dev->dev_private;
2302         bool lists_empty;
2303         int ret;
2304
2305         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2306                        list_empty(&dev_priv->render_ring.active_list) &&
2307                        list_empty(&dev_priv->bsd_ring.active_list) &&
2308                        list_empty(&dev_priv->blt_ring.active_list));
2309         if (lists_empty)
2310                 return 0;
2311
2312         /* Flush everything onto the inactive list. */
2313         ret = i915_ring_idle(dev, &dev_priv->render_ring);
2314         if (ret)
2315                 return ret;
2316
2317         ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2318         if (ret)
2319                 return ret;
2320
2321         ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2322         if (ret)
2323                 return ret;
2324
2325         return 0;
2326 }
2327
2328 static void sandybridge_write_fence_reg(struct drm_gem_object *obj)
2329 {
2330         struct drm_device *dev = obj->dev;
2331         drm_i915_private_t *dev_priv = dev->dev_private;
2332         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2333         u32 size = i915_gem_get_gtt_size(obj_priv);
2334         int regnum = obj_priv->fence_reg;
2335         uint64_t val;
2336
2337         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2338                     0xfffff000) << 32;
2339         val |= obj_priv->gtt_offset & 0xfffff000;
2340         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2341                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2342
2343         if (obj_priv->tiling_mode == I915_TILING_Y)
2344                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2345         val |= I965_FENCE_REG_VALID;
2346
2347         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2348 }
2349
2350 static void i965_write_fence_reg(struct drm_gem_object *obj)
2351 {
2352         struct drm_device *dev = obj->dev;
2353         drm_i915_private_t *dev_priv = dev->dev_private;
2354         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2355         u32 size = i915_gem_get_gtt_size(obj_priv);
2356         int regnum = obj_priv->fence_reg;
2357         uint64_t val;
2358
2359         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2360                     0xfffff000) << 32;
2361         val |= obj_priv->gtt_offset & 0xfffff000;
2362         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2363         if (obj_priv->tiling_mode == I915_TILING_Y)
2364                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2365         val |= I965_FENCE_REG_VALID;
2366
2367         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2368 }
2369
2370 static void i915_write_fence_reg(struct drm_gem_object *obj)
2371 {
2372         struct drm_device *dev = obj->dev;
2373         drm_i915_private_t *dev_priv = dev->dev_private;
2374         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2375         u32 size = i915_gem_get_gtt_size(obj_priv);
2376         uint32_t fence_reg, val, pitch_val;
2377         int tile_width;
2378
2379         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2380             (obj_priv->gtt_offset & (size - 1))) {
2381                 WARN(1, "%s: object 0x%08x [fenceable? %d] not 1M or size (0x%08x) aligned [gtt_space offset=%lx, size=%lx]\n",
2382                      __func__, obj_priv->gtt_offset, obj_priv->fenceable, size,
2383                      obj_priv->gtt_space->start, obj_priv->gtt_space->size);
2384                 return;
2385         }
2386
2387         if (obj_priv->tiling_mode == I915_TILING_Y &&
2388             HAS_128_BYTE_Y_TILING(dev))
2389                 tile_width = 128;
2390         else
2391                 tile_width = 512;
2392
2393         /* Note: pitch better be a power of two tile widths */
2394         pitch_val = obj_priv->stride / tile_width;
2395         pitch_val = ffs(pitch_val) - 1;
2396
2397         if (obj_priv->tiling_mode == I915_TILING_Y &&
2398             HAS_128_BYTE_Y_TILING(dev))
2399                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2400         else
2401                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2402
2403         val = obj_priv->gtt_offset;
2404         if (obj_priv->tiling_mode == I915_TILING_Y)
2405                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2406         val |= I915_FENCE_SIZE_BITS(size);
2407         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2408         val |= I830_FENCE_REG_VALID;
2409
2410         fence_reg = obj_priv->fence_reg;
2411         if (fence_reg < 8)
2412                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2413         else
2414                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2415         I915_WRITE(fence_reg, val);
2416 }
2417
2418 static void i830_write_fence_reg(struct drm_gem_object *obj)
2419 {
2420         struct drm_device *dev = obj->dev;
2421         drm_i915_private_t *dev_priv = dev->dev_private;
2422         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2423         u32 size = i915_gem_get_gtt_size(obj_priv);
2424         int regnum = obj_priv->fence_reg;
2425         uint32_t val;
2426         uint32_t pitch_val;
2427         uint32_t fence_size_bits;
2428
2429         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2430             (obj_priv->gtt_offset & (obj->size - 1))) {
2431                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2432                      __func__, obj_priv->gtt_offset);
2433                 return;
2434         }
2435
2436         pitch_val = obj_priv->stride / 128;
2437         pitch_val = ffs(pitch_val) - 1;
2438         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2439
2440         val = obj_priv->gtt_offset;
2441         if (obj_priv->tiling_mode == I915_TILING_Y)
2442                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2443         fence_size_bits = I830_FENCE_SIZE_BITS(size);
2444         WARN_ON(fence_size_bits & ~0x00000f00);
2445         val |= fence_size_bits;
2446         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2447         val |= I830_FENCE_REG_VALID;
2448
2449         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2450 }
2451
2452 static int i915_find_fence_reg(struct drm_device *dev,
2453                                bool interruptible)
2454 {
2455         struct drm_i915_private *dev_priv = dev->dev_private;
2456         struct drm_i915_fence_reg *reg;
2457         struct drm_i915_gem_object *obj_priv = NULL;
2458         int i, avail, ret;
2459
2460         /* First try to find a free reg */
2461         avail = 0;
2462         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2463                 reg = &dev_priv->fence_regs[i];
2464                 if (!reg->obj)
2465                         return i;
2466
2467                 obj_priv = to_intel_bo(reg->obj);
2468                 if (!obj_priv->pin_count)
2469                     avail++;
2470         }
2471
2472         if (avail == 0)
2473                 return -ENOSPC;
2474
2475         /* None available, try to steal one or wait for a user to finish */
2476         avail = I915_FENCE_REG_NONE;
2477         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2478                             lru_list) {
2479                 obj_priv = to_intel_bo(reg->obj);
2480                 if (obj_priv->pin_count)
2481                         continue;
2482
2483                 /* found one! */
2484                 avail = obj_priv->fence_reg;
2485                 break;
2486         }
2487
2488         BUG_ON(avail == I915_FENCE_REG_NONE);
2489
2490         /* We only have a reference on obj from the active list. put_fence_reg
2491          * might drop that one, causing a use-after-free in it. So hold a
2492          * private reference to obj like the other callers of put_fence_reg
2493          * (set_tiling ioctl) do. */
2494         drm_gem_object_reference(&obj_priv->base);
2495         ret = i915_gem_object_put_fence_reg(&obj_priv->base, interruptible);
2496         drm_gem_object_unreference(&obj_priv->base);
2497         if (ret != 0)
2498                 return ret;
2499
2500         return avail;
2501 }
2502
2503 /**
2504  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2505  * @obj: object to map through a fence reg
2506  *
2507  * When mapping objects through the GTT, userspace wants to be able to write
2508  * to them without having to worry about swizzling if the object is tiled.
2509  *
2510  * This function walks the fence regs looking for a free one for @obj,
2511  * stealing one if it can't find any.
2512  *
2513  * It then sets up the reg based on the object's properties: address, pitch
2514  * and tiling format.
2515  */
2516 int
2517 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2518                               bool interruptible)
2519 {
2520         struct drm_device *dev = obj->dev;
2521         struct drm_i915_private *dev_priv = dev->dev_private;
2522         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2523         struct drm_i915_fence_reg *reg = NULL;
2524         int ret;
2525
2526         /* Just update our place in the LRU if our fence is getting used. */
2527         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2528                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2529                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2530                 return 0;
2531         }
2532
2533         switch (obj_priv->tiling_mode) {
2534         case I915_TILING_NONE:
2535                 WARN(1, "allocating a fence for non-tiled object?\n");
2536                 break;
2537         case I915_TILING_X:
2538                 if (!obj_priv->stride)
2539                         return -EINVAL;
2540                 WARN((obj_priv->stride & (512 - 1)),
2541                      "object 0x%08x is X tiled but has non-512B pitch\n",
2542                      obj_priv->gtt_offset);
2543                 break;
2544         case I915_TILING_Y:
2545                 if (!obj_priv->stride)
2546                         return -EINVAL;
2547                 WARN((obj_priv->stride & (128 - 1)),
2548                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2549                      obj_priv->gtt_offset);
2550                 break;
2551         }
2552
2553         ret = i915_find_fence_reg(dev, interruptible);
2554         if (ret < 0)
2555                 return ret;
2556
2557         obj_priv->fence_reg = ret;
2558         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2559         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2560
2561         reg->obj = obj;
2562
2563         switch (INTEL_INFO(dev)->gen) {
2564         case 6:
2565                 sandybridge_write_fence_reg(obj);
2566                 break;
2567         case 5:
2568         case 4:
2569                 i965_write_fence_reg(obj);
2570                 break;
2571         case 3:
2572                 i915_write_fence_reg(obj);
2573                 break;
2574         case 2:
2575                 i830_write_fence_reg(obj);
2576                 break;
2577         }
2578
2579         trace_i915_gem_object_get_fence(obj,
2580                                         obj_priv->fence_reg,
2581                                         obj_priv->tiling_mode);
2582
2583         return 0;
2584 }
2585
2586 /**
2587  * i915_gem_clear_fence_reg - clear out fence register info
2588  * @obj: object to clear
2589  *
2590  * Zeroes out the fence register itself and clears out the associated
2591  * data structures in dev_priv and obj_priv.
2592  */
2593 static void
2594 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2595 {
2596         struct drm_device *dev = obj->dev;
2597         drm_i915_private_t *dev_priv = dev->dev_private;
2598         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2599         struct drm_i915_fence_reg *reg =
2600                 &dev_priv->fence_regs[obj_priv->fence_reg];
2601         uint32_t fence_reg;
2602
2603         switch (INTEL_INFO(dev)->gen) {
2604         case 6:
2605                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2606                              (obj_priv->fence_reg * 8), 0);
2607                 break;
2608         case 5:
2609         case 4:
2610                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2611                 break;
2612         case 3:
2613                 if (obj_priv->fence_reg >= 8)
2614                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2615                 else
2616         case 2:
2617                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2618
2619                 I915_WRITE(fence_reg, 0);
2620                 break;
2621         }
2622
2623         reg->obj = NULL;
2624         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2625         list_del_init(&reg->lru_list);
2626 }
2627
2628 /**
2629  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2630  * to the buffer to finish, and then resets the fence register.
2631  * @obj: tiled object holding a fence register.
2632  * @bool: whether the wait upon the fence is interruptible
2633  *
2634  * Zeroes out the fence register itself and clears out the associated
2635  * data structures in dev_priv and obj_priv.
2636  */
2637 int
2638 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2639                               bool interruptible)
2640 {
2641         struct drm_device *dev = obj->dev;
2642         struct drm_i915_private *dev_priv = dev->dev_private;
2643         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2644         struct drm_i915_fence_reg *reg;
2645
2646         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2647                 return 0;
2648
2649         /* If we've changed tiling, GTT-mappings of the object
2650          * need to re-fault to ensure that the correct fence register
2651          * setup is in place.
2652          */
2653         i915_gem_release_mmap(obj);
2654
2655         /* On the i915, GPU access to tiled buffers is via a fence,
2656          * therefore we must wait for any outstanding access to complete
2657          * before clearing the fence.
2658          */
2659         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2660         if (reg->gpu) {
2661                 int ret;
2662
2663                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2664                 if (ret)
2665                         return ret;
2666
2667                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2668                 if (ret)
2669                         return ret;
2670
2671                 reg->gpu = false;
2672         }
2673
2674         i915_gem_object_flush_gtt_write_domain(obj);
2675         i915_gem_clear_fence_reg(obj);
2676
2677         return 0;
2678 }
2679
2680 /**
2681  * Finds free space in the GTT aperture and binds the object there.
2682  */
2683 static int
2684 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
2685                             unsigned alignment,
2686                             bool mappable,
2687                             bool need_fence)
2688 {
2689         struct drm_device *dev = obj->dev;
2690         drm_i915_private_t *dev_priv = dev->dev_private;
2691         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2692         struct drm_mm_node *free_space;
2693         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2694         u32 size, fence_size, fence_alignment;
2695         int ret;
2696
2697         if (obj_priv->madv != I915_MADV_WILLNEED) {
2698                 DRM_ERROR("Attempting to bind a purgeable object\n");
2699                 return -EINVAL;
2700         }
2701
2702         fence_size = i915_gem_get_gtt_size(obj_priv);
2703         fence_alignment = i915_gem_get_gtt_alignment(obj_priv);
2704
2705         if (alignment == 0)
2706                 alignment = need_fence ? fence_alignment : 4096;
2707         if (need_fence && alignment & (fence_alignment - 1)) {
2708                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2709                 return -EINVAL;
2710         }
2711
2712         size = need_fence ? fence_size : obj->size;
2713
2714         /* If the object is bigger than the entire aperture, reject it early
2715          * before evicting everything in a vain attempt to find space.
2716          */
2717         if (obj->size >
2718             (mappable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2719                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2720                 return -E2BIG;
2721         }
2722
2723  search_free:
2724         if (mappable)
2725                 free_space =
2726                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2727                                                     size, alignment, 0,
2728                                                     dev_priv->mm.gtt_mappable_end,
2729                                                     0);
2730         else
2731                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2732                                                 size, alignment, 0);
2733
2734         if (free_space != NULL) {
2735                 if (mappable)
2736                         obj_priv->gtt_space =
2737                                 drm_mm_get_block_range_generic(free_space,
2738                                                                size, alignment, 0,
2739                                                                dev_priv->mm.gtt_mappable_end,
2740                                                                0);
2741                 else
2742                         obj_priv->gtt_space =
2743                                 drm_mm_get_block(free_space, size, alignment);
2744         }
2745         if (obj_priv->gtt_space == NULL) {
2746                 /* If the gtt is empty and we're still having trouble
2747                  * fitting our object in, we're out of memory.
2748                  */
2749                 ret = i915_gem_evict_something(dev, size, alignment, mappable);
2750                 if (ret)
2751                         return ret;
2752
2753                 goto search_free;
2754         }
2755
2756         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2757         if (ret) {
2758                 drm_mm_put_block(obj_priv->gtt_space);
2759                 obj_priv->gtt_space = NULL;
2760
2761                 if (ret == -ENOMEM) {
2762                         /* first try to clear up some space from the GTT */
2763                         ret = i915_gem_evict_something(dev, size,
2764                                                        alignment, mappable);
2765                         if (ret) {
2766                                 /* now try to shrink everyone else */
2767                                 if (gfpmask) {
2768                                         gfpmask = 0;
2769                                         goto search_free;
2770                                 }
2771
2772                                 return ret;
2773                         }
2774
2775                         goto search_free;
2776                 }
2777
2778                 return ret;
2779         }
2780
2781         /* Create an AGP memory structure pointing at our pages, and bind it
2782          * into the GTT.
2783          */
2784         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2785                                                obj_priv->pages,
2786                                                obj->size >> PAGE_SHIFT,
2787                                                obj_priv->gtt_space->start,
2788                                                obj_priv->agp_type);
2789         if (obj_priv->agp_mem == NULL) {
2790                 i915_gem_object_put_pages_gtt(obj);
2791                 drm_mm_put_block(obj_priv->gtt_space);
2792                 obj_priv->gtt_space = NULL;
2793
2794                 ret = i915_gem_evict_something(dev, size,
2795                                                alignment, mappable);
2796                 if (ret)
2797                         return ret;
2798
2799                 goto search_free;
2800         }
2801
2802         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2803
2804         /* keep track of bounds object by adding it to the inactive list */
2805         list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
2806         i915_gem_info_add_gtt(dev_priv, obj_priv);
2807
2808         /* Assert that the object is not currently in any GPU domain. As it
2809          * wasn't in the GTT, there shouldn't be any way it could have been in
2810          * a GPU cache
2811          */
2812         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2813         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2814
2815         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset, mappable);
2816
2817         obj_priv->fenceable =
2818                 obj_priv->gtt_space->size == fence_size &&
2819                 (obj_priv->gtt_space->start & (fence_alignment -1)) == 0;
2820
2821         obj_priv->mappable =
2822                 obj_priv->gtt_offset + obj->size <= dev_priv->mm.gtt_mappable_end;
2823
2824         return 0;
2825 }
2826
2827 void
2828 i915_gem_clflush_object(struct drm_gem_object *obj)
2829 {
2830         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2831
2832         /* If we don't have a page list set up, then we're not pinned
2833          * to GPU, and we can ignore the cache flush because it'll happen
2834          * again at bind time.
2835          */
2836         if (obj_priv->pages == NULL)
2837                 return;
2838
2839         trace_i915_gem_object_clflush(obj);
2840
2841         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2842 }
2843
2844 /** Flushes any GPU write domain for the object if it's dirty. */
2845 static int
2846 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2847                                        bool pipelined)
2848 {
2849         struct drm_device *dev = obj->dev;
2850
2851         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2852                 return 0;
2853
2854         /* Queue the GPU write cache flushing we need. */
2855         i915_gem_flush_ring(dev, NULL,
2856                             to_intel_bo(obj)->ring,
2857                             0, obj->write_domain);
2858         BUG_ON(obj->write_domain);
2859
2860         if (pipelined)
2861                 return 0;
2862
2863         return i915_gem_object_wait_rendering(obj, true);
2864 }
2865
2866 /** Flushes the GTT write domain for the object if it's dirty. */
2867 static void
2868 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2869 {
2870         uint32_t old_write_domain;
2871
2872         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2873                 return;
2874
2875         /* No actual flushing is required for the GTT write domain.   Writes
2876          * to it immediately go to main memory as far as we know, so there's
2877          * no chipset flush.  It also doesn't land in render cache.
2878          */
2879         i915_gem_release_mmap(obj);
2880
2881         old_write_domain = obj->write_domain;
2882         obj->write_domain = 0;
2883
2884         trace_i915_gem_object_change_domain(obj,
2885                                             obj->read_domains,
2886                                             old_write_domain);
2887 }
2888
2889 /** Flushes the CPU write domain for the object if it's dirty. */
2890 static void
2891 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2892 {
2893         struct drm_device *dev = obj->dev;
2894         uint32_t old_write_domain;
2895
2896         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2897                 return;
2898
2899         i915_gem_clflush_object(obj);
2900         drm_agp_chipset_flush(dev);
2901         old_write_domain = obj->write_domain;
2902         obj->write_domain = 0;
2903
2904         trace_i915_gem_object_change_domain(obj,
2905                                             obj->read_domains,
2906                                             old_write_domain);
2907 }
2908
2909 /**
2910  * Moves a single object to the GTT read, and possibly write domain.
2911  *
2912  * This function returns when the move is complete, including waiting on
2913  * flushes to occur.
2914  */
2915 int
2916 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2917 {
2918         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2919         uint32_t old_write_domain, old_read_domains;
2920         int ret;
2921
2922         /* Not valid to be called on unbound objects. */
2923         if (obj_priv->gtt_space == NULL)
2924                 return -EINVAL;
2925
2926         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2927         if (ret != 0)
2928                 return ret;
2929
2930         i915_gem_object_flush_cpu_write_domain(obj);
2931
2932         if (write) {
2933                 ret = i915_gem_object_wait_rendering(obj, true);
2934                 if (ret)
2935                         return ret;
2936         }
2937
2938         old_write_domain = obj->write_domain;
2939         old_read_domains = obj->read_domains;
2940
2941         /* It should now be out of any other write domains, and we can update
2942          * the domain values for our changes.
2943          */
2944         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2945         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2946         if (write) {
2947                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2948                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2949                 obj_priv->dirty = 1;
2950         }
2951
2952         trace_i915_gem_object_change_domain(obj,
2953                                             old_read_domains,
2954                                             old_write_domain);
2955
2956         return 0;
2957 }
2958
2959 /*
2960  * Prepare buffer for display plane. Use uninterruptible for possible flush
2961  * wait, as in modesetting process we're not supposed to be interrupted.
2962  */
2963 int
2964 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2965                                      bool pipelined)
2966 {
2967         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2968         uint32_t old_read_domains;
2969         int ret;
2970
2971         /* Not valid to be called on unbound objects. */
2972         if (obj_priv->gtt_space == NULL)
2973                 return -EINVAL;
2974
2975         ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2976         if (ret)
2977                 return ret;
2978
2979         /* Currently, we are always called from an non-interruptible context. */
2980         if (!pipelined) {
2981                 ret = i915_gem_object_wait_rendering(obj, false);
2982                 if (ret)
2983                         return ret;
2984         }
2985
2986         i915_gem_object_flush_cpu_write_domain(obj);
2987
2988         old_read_domains = obj->read_domains;
2989         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2990
2991         trace_i915_gem_object_change_domain(obj,
2992                                             old_read_domains,
2993                                             obj->write_domain);
2994
2995         return 0;
2996 }
2997
2998 /**
2999  * Moves a single object to the CPU read, and possibly write domain.
3000  *
3001  * This function returns when the move is complete, including waiting on
3002  * flushes to occur.
3003  */
3004 static int
3005 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
3006 {
3007         uint32_t old_write_domain, old_read_domains;
3008         int ret;
3009
3010         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3011         if (ret != 0)
3012                 return ret;
3013
3014         i915_gem_object_flush_gtt_write_domain(obj);
3015
3016         /* If we have a partially-valid cache of the object in the CPU,
3017          * finish invalidating it and free the per-page flags.
3018          */
3019         i915_gem_object_set_to_full_cpu_read_domain(obj);
3020
3021         if (write) {
3022                 ret = i915_gem_object_wait_rendering(obj, true);
3023                 if (ret)
3024                         return ret;
3025         }
3026
3027         old_write_domain = obj->write_domain;
3028         old_read_domains = obj->read_domains;
3029
3030         /* Flush the CPU cache if it's still invalid. */
3031         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3032                 i915_gem_clflush_object(obj);
3033
3034                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3035         }
3036
3037         /* It should now be out of any other write domains, and we can update
3038          * the domain values for our changes.
3039          */
3040         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3041
3042         /* If we're writing through the CPU, then the GPU read domains will
3043          * need to be invalidated at next use.
3044          */
3045         if (write) {
3046                 obj->read_domains = I915_GEM_DOMAIN_CPU;
3047                 obj->write_domain = I915_GEM_DOMAIN_CPU;
3048         }
3049
3050         trace_i915_gem_object_change_domain(obj,
3051                                             old_read_domains,
3052                                             old_write_domain);
3053
3054         return 0;
3055 }
3056
3057 /*
3058  * Set the next domain for the specified object. This
3059  * may not actually perform the necessary flushing/invaliding though,
3060  * as that may want to be batched with other set_domain operations
3061  *
3062  * This is (we hope) the only really tricky part of gem. The goal
3063  * is fairly simple -- track which caches hold bits of the object
3064  * and make sure they remain coherent. A few concrete examples may
3065  * help to explain how it works. For shorthand, we use the notation
3066  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3067  * a pair of read and write domain masks.
3068  *
3069  * Case 1: the batch buffer
3070  *
3071  *      1. Allocated
3072  *      2. Written by CPU
3073  *      3. Mapped to GTT
3074  *      4. Read by GPU
3075  *      5. Unmapped from GTT
3076  *      6. Freed
3077  *
3078  *      Let's take these a step at a time
3079  *
3080  *      1. Allocated
3081  *              Pages allocated from the kernel may still have
3082  *              cache contents, so we set them to (CPU, CPU) always.
3083  *      2. Written by CPU (using pwrite)
3084  *              The pwrite function calls set_domain (CPU, CPU) and
3085  *              this function does nothing (as nothing changes)
3086  *      3. Mapped by GTT
3087  *              This function asserts that the object is not
3088  *              currently in any GPU-based read or write domains
3089  *      4. Read by GPU
3090  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3091  *              As write_domain is zero, this function adds in the
3092  *              current read domains (CPU+COMMAND, 0).
3093  *              flush_domains is set to CPU.
3094  *              invalidate_domains is set to COMMAND
3095  *              clflush is run to get data out of the CPU caches
3096  *              then i915_dev_set_domain calls i915_gem_flush to
3097  *              emit an MI_FLUSH and drm_agp_chipset_flush
3098  *      5. Unmapped from GTT
3099  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3100  *              flush_domains and invalidate_domains end up both zero
3101  *              so no flushing/invalidating happens
3102  *      6. Freed
3103  *              yay, done
3104  *
3105  * Case 2: The shared render buffer
3106  *
3107  *      1. Allocated
3108  *      2. Mapped to GTT
3109  *      3. Read/written by GPU
3110  *      4. set_domain to (CPU,CPU)
3111  *      5. Read/written by CPU
3112  *      6. Read/written by GPU
3113  *
3114  *      1. Allocated
3115  *              Same as last example, (CPU, CPU)
3116  *      2. Mapped to GTT
3117  *              Nothing changes (assertions find that it is not in the GPU)
3118  *      3. Read/written by GPU
3119  *              execbuffer calls set_domain (RENDER, RENDER)
3120  *              flush_domains gets CPU
3121  *              invalidate_domains gets GPU
3122  *              clflush (obj)
3123  *              MI_FLUSH and drm_agp_chipset_flush
3124  *      4. set_domain (CPU, CPU)
3125  *              flush_domains gets GPU
3126  *              invalidate_domains gets CPU
3127  *              wait_rendering (obj) to make sure all drawing is complete.
3128  *              This will include an MI_FLUSH to get the data from GPU
3129  *              to memory
3130  *              clflush (obj) to invalidate the CPU cache
3131  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3132  *      5. Read/written by CPU
3133  *              cache lines are loaded and dirtied
3134  *      6. Read written by GPU
3135  *              Same as last GPU access
3136  *
3137  * Case 3: The constant buffer
3138  *
3139  *      1. Allocated
3140  *      2. Written by CPU
3141  *      3. Read by GPU
3142  *      4. Updated (written) by CPU again
3143  *      5. Read by GPU
3144  *
3145  *      1. Allocated
3146  *              (CPU, CPU)
3147  *      2. Written by CPU
3148  *              (CPU, CPU)
3149  *      3. Read by GPU
3150  *              (CPU+RENDER, 0)
3151  *              flush_domains = CPU
3152  *              invalidate_domains = RENDER
3153  *              clflush (obj)
3154  *              MI_FLUSH
3155  *              drm_agp_chipset_flush
3156  *      4. Updated (written) by CPU again
3157  *              (CPU, CPU)
3158  *              flush_domains = 0 (no previous write domain)
3159  *              invalidate_domains = 0 (no new read domains)
3160  *      5. Read by GPU
3161  *              (CPU+RENDER, 0)
3162  *              flush_domains = CPU
3163  *              invalidate_domains = RENDER
3164  *              clflush (obj)
3165  *              MI_FLUSH
3166  *              drm_agp_chipset_flush
3167  */
3168 static void
3169 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3170                                   struct intel_ring_buffer *ring)
3171 {
3172         struct drm_device               *dev = obj->dev;
3173         struct drm_i915_private         *dev_priv = dev->dev_private;
3174         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3175         uint32_t                        invalidate_domains = 0;
3176         uint32_t                        flush_domains = 0;
3177
3178         /*
3179          * If the object isn't moving to a new write domain,
3180          * let the object stay in multiple read domains
3181          */
3182         if (obj->pending_write_domain == 0)
3183                 obj->pending_read_domains |= obj->read_domains;
3184
3185         /*
3186          * Flush the current write domain if
3187          * the new read domains don't match. Invalidate
3188          * any read domains which differ from the old
3189          * write domain
3190          */
3191         if (obj->write_domain &&
3192             (obj->write_domain != obj->pending_read_domains ||
3193              obj_priv->ring != ring)) {
3194                 flush_domains |= obj->write_domain;
3195                 invalidate_domains |=
3196                         obj->pending_read_domains & ~obj->write_domain;
3197         }
3198         /*
3199          * Invalidate any read caches which may have
3200          * stale data. That is, any new read domains.
3201          */
3202         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3203         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3204                 i915_gem_clflush_object(obj);
3205
3206         /* blow away mappings if mapped through GTT */
3207         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
3208                 i915_gem_release_mmap(obj);
3209
3210         /* The actual obj->write_domain will be updated with
3211          * pending_write_domain after we emit the accumulated flush for all
3212          * of our domain changes in execbuffers (which clears objects'
3213          * write_domains).  So if we have a current write domain that we
3214          * aren't changing, set pending_write_domain to that.
3215          */
3216         if (flush_domains == 0 && obj->pending_write_domain == 0)
3217                 obj->pending_write_domain = obj->write_domain;
3218
3219         dev->invalidate_domains |= invalidate_domains;
3220         dev->flush_domains |= flush_domains;
3221         if (flush_domains & I915_GEM_GPU_DOMAINS)
3222                 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3223         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3224                 dev_priv->mm.flush_rings |= ring->id;
3225 }
3226
3227 /**
3228  * Moves the object from a partially CPU read to a full one.
3229  *
3230  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3231  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3232  */
3233 static void
3234 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3235 {
3236         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3237
3238         if (!obj_priv->page_cpu_valid)
3239                 return;
3240
3241         /* If we're partially in the CPU read domain, finish moving it in.
3242          */
3243         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3244                 int i;
3245
3246                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3247                         if (obj_priv->page_cpu_valid[i])
3248                                 continue;
3249                         drm_clflush_pages(obj_priv->pages + i, 1);
3250                 }
3251         }
3252
3253         /* Free the page_cpu_valid mappings which are now stale, whether
3254          * or not we've got I915_GEM_DOMAIN_CPU.
3255          */
3256         kfree(obj_priv->page_cpu_valid);
3257         obj_priv->page_cpu_valid = NULL;
3258 }
3259
3260 /**
3261  * Set the CPU read domain on a range of the object.
3262  *
3263  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3264  * not entirely valid.  The page_cpu_valid member of the object flags which
3265  * pages have been flushed, and will be respected by
3266  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3267  * of the whole object.
3268  *
3269  * This function returns when the move is complete, including waiting on
3270  * flushes to occur.
3271  */
3272 static int
3273 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3274                                           uint64_t offset, uint64_t size)
3275 {
3276         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3277         uint32_t old_read_domains;
3278         int i, ret;
3279
3280         if (offset == 0 && size == obj->size)
3281                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3282
3283         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3284         if (ret != 0)
3285                 return ret;
3286         i915_gem_object_flush_gtt_write_domain(obj);
3287
3288         /* If we're already fully in the CPU read domain, we're done. */
3289         if (obj_priv->page_cpu_valid == NULL &&
3290             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3291                 return 0;
3292
3293         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3294          * newly adding I915_GEM_DOMAIN_CPU
3295          */
3296         if (obj_priv->page_cpu_valid == NULL) {
3297                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3298                                                    GFP_KERNEL);
3299                 if (obj_priv->page_cpu_valid == NULL)
3300                         return -ENOMEM;
3301         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3302                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3303
3304         /* Flush the cache on any pages that are still invalid from the CPU's
3305          * perspective.
3306          */
3307         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3308              i++) {
3309                 if (obj_priv->page_cpu_valid[i])
3310                         continue;
3311
3312                 drm_clflush_pages(obj_priv->pages + i, 1);
3313
3314                 obj_priv->page_cpu_valid[i] = 1;
3315         }
3316
3317         /* It should now be out of any other write domains, and we can update
3318          * the domain values for our changes.
3319          */
3320         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3321
3322         old_read_domains = obj->read_domains;
3323         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3324
3325         trace_i915_gem_object_change_domain(obj,
3326                                             old_read_domains,
3327                                             obj->write_domain);
3328
3329         return 0;
3330 }
3331
3332 /**
3333  * Pin an object to the GTT and evaluate the relocations landing in it.
3334  */
3335 static int
3336 i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
3337                              struct drm_file *file_priv,
3338                              struct drm_i915_gem_exec_object2 *entry)
3339 {
3340         struct drm_device *dev = obj->base.dev;
3341         drm_i915_private_t *dev_priv = dev->dev_private;
3342         struct drm_i915_gem_relocation_entry __user *user_relocs;
3343         struct drm_gem_object *target_obj = NULL;
3344         uint32_t target_handle = 0;
3345         int i, ret = 0;
3346
3347         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3348         for (i = 0; i < entry->relocation_count; i++) {
3349                 struct drm_i915_gem_relocation_entry reloc;
3350                 uint32_t target_offset;
3351
3352                 if (__copy_from_user_inatomic(&reloc,
3353                                               user_relocs+i,
3354                                               sizeof(reloc))) {
3355                         ret = -EFAULT;
3356                         break;
3357                 }
3358
3359                 if (reloc.target_handle != target_handle) {
3360                         drm_gem_object_unreference(target_obj);
3361
3362                         target_obj = drm_gem_object_lookup(dev, file_priv,
3363                                                            reloc.target_handle);
3364                         if (target_obj == NULL) {
3365                                 ret = -ENOENT;
3366                                 break;
3367                         }
3368
3369                         target_handle = reloc.target_handle;
3370                 }
3371                 target_offset = to_intel_bo(target_obj)->gtt_offset;
3372
3373 #if WATCH_RELOC
3374                 DRM_INFO("%s: obj %p offset %08x target %d "
3375                          "read %08x write %08x gtt %08x "
3376                          "presumed %08x delta %08x\n",
3377                          __func__,
3378                          obj,
3379                          (int) reloc.offset,
3380                          (int) reloc.target_handle,
3381                          (int) reloc.read_domains,
3382                          (int) reloc.write_domain,
3383                          (int) target_offset,
3384                          (int) reloc.presumed_offset,
3385                          reloc.delta);
3386 #endif
3387
3388                 /* The target buffer should have appeared before us in the
3389                  * exec_object list, so it should have a GTT space bound by now.
3390                  */
3391                 if (target_offset == 0) {
3392                         DRM_ERROR("No GTT space found for object %d\n",
3393                                   reloc.target_handle);
3394                         ret = -EINVAL;
3395                         break;
3396                 }
3397
3398                 /* Validate that the target is in a valid r/w GPU domain */
3399                 if (reloc.write_domain & (reloc.write_domain - 1)) {
3400                         DRM_ERROR("reloc with multiple write domains: "
3401                                   "obj %p target %d offset %d "
3402                                   "read %08x write %08x",
3403                                   obj, reloc.target_handle,
3404                                   (int) reloc.offset,
3405                                   reloc.read_domains,
3406                                   reloc.write_domain);
3407                         ret = -EINVAL;
3408                         break;
3409                 }
3410                 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3411                     reloc.read_domains & I915_GEM_DOMAIN_CPU) {
3412                         DRM_ERROR("reloc with read/write CPU domains: "
3413                                   "obj %p target %d offset %d "
3414                                   "read %08x write %08x",
3415                                   obj, reloc.target_handle,
3416                                   (int) reloc.offset,
3417                                   reloc.read_domains,
3418                                   reloc.write_domain);
3419                         ret = -EINVAL;
3420                         break;
3421                 }
3422                 if (reloc.write_domain && target_obj->pending_write_domain &&
3423                     reloc.write_domain != target_obj->pending_write_domain) {
3424                         DRM_ERROR("Write domain conflict: "
3425                                   "obj %p target %d offset %d "
3426                                   "new %08x old %08x\n",
3427                                   obj, reloc.target_handle,
3428                                   (int) reloc.offset,
3429                                   reloc.write_domain,
3430                                   target_obj->pending_write_domain);
3431                         ret = -EINVAL;
3432                         break;
3433                 }
3434
3435                 target_obj->pending_read_domains |= reloc.read_domains;
3436                 target_obj->pending_write_domain |= reloc.write_domain;
3437
3438                 /* If the relocation already has the right value in it, no
3439                  * more work needs to be done.
3440                  */
3441                 if (target_offset == reloc.presumed_offset)
3442                         continue;
3443
3444                 /* Check that the relocation address is valid... */
3445                 if (reloc.offset > obj->base.size - 4) {
3446                         DRM_ERROR("Relocation beyond object bounds: "
3447                                   "obj %p target %d offset %d size %d.\n",
3448                                   obj, reloc.target_handle,
3449                                   (int) reloc.offset, (int) obj->base.size);
3450                         ret = -EINVAL;
3451                         break;
3452                 }
3453                 if (reloc.offset & 3) {
3454                         DRM_ERROR("Relocation not 4-byte aligned: "
3455                                   "obj %p target %d offset %d.\n",
3456                                   obj, reloc.target_handle,
3457                                   (int) reloc.offset);
3458                         ret = -EINVAL;
3459                         break;
3460                 }
3461
3462                 /* and points to somewhere within the target object. */
3463                 if (reloc.delta >= target_obj->size) {
3464                         DRM_ERROR("Relocation beyond target object bounds: "
3465                                   "obj %p target %d delta %d size %d.\n",
3466                                   obj, reloc.target_handle,
3467                                   (int) reloc.delta, (int) target_obj->size);
3468                         ret = -EINVAL;
3469                         break;
3470                 }
3471
3472                 reloc.delta += target_offset;
3473                 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3474                         uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3475                         char *vaddr;
3476
3477                         vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT]);
3478                         *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3479                         kunmap_atomic(vaddr);
3480                 } else {
3481                         uint32_t __iomem *reloc_entry;
3482                         void __iomem *reloc_page;
3483
3484                         ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3485                         if (ret)
3486                                 break;
3487
3488                         /* Map the page containing the relocation we're going to perform.  */
3489                         reloc.offset += obj->gtt_offset;
3490                         reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3491                                                               reloc.offset & PAGE_MASK);
3492                         reloc_entry = (uint32_t __iomem *)
3493                                 (reloc_page + (reloc.offset & ~PAGE_MASK));
3494                         iowrite32(reloc.delta, reloc_entry);
3495                         io_mapping_unmap_atomic(reloc_page);
3496                 }
3497
3498                 /* and update the user's relocation entry */
3499                 reloc.presumed_offset = target_offset;
3500                 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3501                                               &reloc.presumed_offset,
3502                                               sizeof(reloc.presumed_offset))) {
3503                     ret = -EFAULT;
3504                     break;
3505                 }
3506         }
3507
3508         drm_gem_object_unreference(target_obj);
3509         return ret;
3510 }
3511
3512 static int
3513 i915_gem_execbuffer_pin(struct drm_device *dev,
3514                         struct drm_file *file,
3515                         struct drm_gem_object **object_list,
3516                         struct drm_i915_gem_exec_object2 *exec_list,
3517                         int count)
3518 {
3519         struct drm_i915_private *dev_priv = dev->dev_private;
3520         int ret, i, retry;
3521
3522         /* attempt to pin all of the buffers into the GTT */
3523         retry = 0;
3524         do {
3525                 ret = 0;
3526                 for (i = 0; i < count; i++) {
3527                         struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3528                         struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3529                         bool need_fence =
3530                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3531                                 obj->tiling_mode != I915_TILING_NONE;
3532
3533                         /* g33/pnv can't fence buffers in the unmappable part */
3534                         bool need_mappable =
3535                                 entry->relocation_count ? true : need_fence;
3536
3537                         /* Check fence reg constraints and rebind if necessary */
3538                         if ((need_fence && !obj->fenceable) ||
3539                             (need_mappable && !obj->mappable)) {
3540                                 ret = i915_gem_object_unbind(&obj->base);
3541                                 if (ret)
3542                                         break;
3543                         }
3544
3545                         ret = i915_gem_object_pin(&obj->base,
3546                                                   entry->alignment,
3547                                                   need_mappable,
3548                                                   need_fence);
3549                         if (ret)
3550                                 break;
3551
3552                         /*
3553                          * Pre-965 chips need a fence register set up in order
3554                          * to properly handle blits to/from tiled surfaces.
3555                          */
3556                         if (need_fence) {
3557                                 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3558                                 if (ret) {
3559                                         i915_gem_object_unpin(&obj->base);
3560                                         break;
3561                                 }
3562
3563                                 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3564                         }
3565
3566                         entry->offset = obj->gtt_offset;
3567                 }
3568
3569                 while (i--)
3570                         i915_gem_object_unpin(object_list[i]);
3571
3572                 if (ret != -ENOSPC || retry > 1)
3573                         return ret;
3574
3575                 /* First attempt, just clear anything that is purgeable.
3576                  * Second attempt, clear the entire GTT.
3577                  */
3578                 ret = i915_gem_evict_everything(dev, retry == 0);
3579                 if (ret)
3580                         return ret;
3581
3582                 retry++;
3583         } while (1);
3584 }
3585
3586 static int
3587 i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3588                                 struct drm_file *file,
3589                                 struct intel_ring_buffer *ring,
3590                                 struct drm_gem_object **objects,
3591                                 int count)
3592 {
3593         struct drm_i915_private *dev_priv = dev->dev_private;
3594         int ret, i;
3595
3596         /* Zero the global flush/invalidate flags. These
3597          * will be modified as new domains are computed
3598          * for each object
3599          */
3600         dev->invalidate_domains = 0;
3601         dev->flush_domains = 0;
3602         dev_priv->mm.flush_rings = 0;
3603         for (i = 0; i < count; i++)
3604                 i915_gem_object_set_to_gpu_domain(objects[i], ring);
3605
3606         if (dev->invalidate_domains | dev->flush_domains) {
3607 #if WATCH_EXEC
3608                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3609                           __func__,
3610                          dev->invalidate_domains,
3611                          dev->flush_domains);
3612 #endif
3613                 i915_gem_flush(dev, file,
3614                                dev->invalidate_domains,
3615                                dev->flush_domains,
3616                                dev_priv->mm.flush_rings);
3617         }
3618
3619         for (i = 0; i < count; i++) {
3620                 struct drm_i915_gem_object *obj = to_intel_bo(objects[i]);
3621                 /* XXX replace with semaphores */
3622                 if (obj->ring && ring != obj->ring) {
3623                         ret = i915_gem_object_wait_rendering(&obj->base, true);
3624                         if (ret)
3625                                 return ret;
3626                 }
3627         }
3628
3629         return 0;
3630 }
3631
3632 /* Throttle our rendering by waiting until the ring has completed our requests
3633  * emitted over 20 msec ago.
3634  *
3635  * Note that if we were to use the current jiffies each time around the loop,
3636  * we wouldn't escape the function with any frames outstanding if the time to
3637  * render a frame was over 20ms.
3638  *
3639  * This should get us reasonable parallelism between CPU and GPU but also
3640  * relatively low latency when blocking on a particular request to finish.
3641  */
3642 static int
3643 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3644 {
3645         struct drm_i915_private *dev_priv = dev->dev_private;
3646         struct drm_i915_file_private *file_priv = file->driver_priv;
3647         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3648         struct drm_i915_gem_request *request;
3649         struct intel_ring_buffer *ring = NULL;
3650         u32 seqno = 0;
3651         int ret;
3652
3653         spin_lock(&file_priv->mm.lock);
3654         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3655                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3656                         break;
3657
3658                 ring = request->ring;
3659                 seqno = request->seqno;
3660         }
3661         spin_unlock(&file_priv->mm.lock);
3662
3663         if (seqno == 0)
3664                 return 0;
3665
3666         ret = 0;
3667         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3668                 /* And wait for the seqno passing without holding any locks and
3669                  * causing extra latency for others. This is safe as the irq
3670                  * generation is designed to be run atomically and so is
3671                  * lockless.
3672                  */
3673                 ring->user_irq_get(ring);
3674                 ret = wait_event_interruptible(ring->irq_queue,
3675                                                i915_seqno_passed(ring->get_seqno(ring), seqno)
3676                                                || atomic_read(&dev_priv->mm.wedged));
3677                 ring->user_irq_put(ring);
3678
3679                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3680                         ret = -EIO;
3681         }
3682
3683         if (ret == 0)
3684                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3685
3686         return ret;
3687 }
3688
3689 static int
3690 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3691                           uint64_t exec_offset)
3692 {
3693         uint32_t exec_start, exec_len;
3694
3695         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3696         exec_len = (uint32_t) exec->batch_len;
3697
3698         if ((exec_start | exec_len) & 0x7)
3699                 return -EINVAL;
3700
3701         if (!exec_start)
3702                 return -EINVAL;
3703
3704         return 0;
3705 }
3706
3707 static int
3708 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3709                    int count)
3710 {
3711         int i;
3712
3713         for (i = 0; i < count; i++) {
3714                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3715                 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
3716
3717                 if (!access_ok(VERIFY_READ, ptr, length))
3718                         return -EFAULT;
3719
3720                 /* we may also need to update the presumed offsets */
3721                 if (!access_ok(VERIFY_WRITE, ptr, length))
3722                         return -EFAULT;
3723
3724                 if (fault_in_pages_readable(ptr, length))
3725                         return -EFAULT;
3726         }
3727
3728         return 0;
3729 }
3730
3731 static int
3732 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3733                        struct drm_file *file,
3734                        struct drm_i915_gem_execbuffer2 *args,
3735                        struct drm_i915_gem_exec_object2 *exec_list)
3736 {
3737         drm_i915_private_t *dev_priv = dev->dev_private;
3738         struct drm_gem_object **object_list = NULL;
3739         struct drm_gem_object *batch_obj;
3740         struct drm_clip_rect *cliprects = NULL;
3741         struct drm_i915_gem_request *request = NULL;
3742         int ret, i, flips;
3743         uint64_t exec_offset;
3744
3745         struct intel_ring_buffer *ring = NULL;
3746
3747         ret = i915_gem_check_is_wedged(dev);
3748         if (ret)
3749                 return ret;
3750
3751         ret = validate_exec_list(exec_list, args->buffer_count);
3752         if (ret)
3753                 return ret;
3754
3755 #if WATCH_EXEC
3756         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3757                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3758 #endif
3759         switch (args->flags & I915_EXEC_RING_MASK) {
3760         case I915_EXEC_DEFAULT:
3761         case I915_EXEC_RENDER:
3762                 ring = &dev_priv->render_ring;
3763                 break;
3764         case I915_EXEC_BSD:
3765                 if (!HAS_BSD(dev)) {
3766                         DRM_ERROR("execbuf with invalid ring (BSD)\n");
3767                         return -EINVAL;
3768                 }
3769                 ring = &dev_priv->bsd_ring;
3770                 break;
3771         case I915_EXEC_BLT:
3772                 if (!HAS_BLT(dev)) {
3773                         DRM_ERROR("execbuf with invalid ring (BLT)\n");
3774                         return -EINVAL;
3775                 }
3776                 ring = &dev_priv->blt_ring;
3777                 break;
3778         default:
3779                 DRM_ERROR("execbuf with unknown ring: %d\n",
3780                           (int)(args->flags & I915_EXEC_RING_MASK));
3781                 return -EINVAL;
3782         }
3783
3784         if (args->buffer_count < 1) {
3785                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3786                 return -EINVAL;
3787         }
3788         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3789         if (object_list == NULL) {
3790                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3791                           args->buffer_count);
3792                 ret = -ENOMEM;
3793                 goto pre_mutex_err;
3794         }
3795
3796         if (args->num_cliprects != 0) {
3797                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3798                                     GFP_KERNEL);
3799                 if (cliprects == NULL) {
3800                         ret = -ENOMEM;
3801                         goto pre_mutex_err;
3802                 }
3803
3804                 ret = copy_from_user(cliprects,
3805                                      (struct drm_clip_rect __user *)
3806                                      (uintptr_t) args->cliprects_ptr,
3807                                      sizeof(*cliprects) * args->num_cliprects);
3808                 if (ret != 0) {
3809                         DRM_ERROR("copy %d cliprects failed: %d\n",
3810                                   args->num_cliprects, ret);
3811                         ret = -EFAULT;
3812                         goto pre_mutex_err;
3813                 }
3814         }
3815
3816         request = kzalloc(sizeof(*request), GFP_KERNEL);
3817         if (request == NULL) {
3818                 ret = -ENOMEM;
3819                 goto pre_mutex_err;
3820         }
3821
3822         ret = i915_mutex_lock_interruptible(dev);
3823         if (ret)
3824                 goto pre_mutex_err;
3825
3826         if (dev_priv->mm.suspended) {
3827                 mutex_unlock(&dev->struct_mutex);
3828                 ret = -EBUSY;
3829                 goto pre_mutex_err;
3830         }
3831
3832         /* Look up object handles */
3833         for (i = 0; i < args->buffer_count; i++) {
3834                 struct drm_i915_gem_object *obj_priv;
3835
3836                 object_list[i] = drm_gem_object_lookup(dev, file,
3837                                                        exec_list[i].handle);
3838                 if (object_list[i] == NULL) {
3839                         DRM_ERROR("Invalid object handle %d at index %d\n",
3840                                    exec_list[i].handle, i);
3841                         /* prevent error path from reading uninitialized data */
3842                         args->buffer_count = i + 1;
3843                         ret = -ENOENT;
3844                         goto err;
3845                 }
3846
3847                 obj_priv = to_intel_bo(object_list[i]);
3848                 if (obj_priv->in_execbuffer) {
3849                         DRM_ERROR("Object %p appears more than once in object list\n",
3850                                    object_list[i]);
3851                         /* prevent error path from reading uninitialized data */
3852                         args->buffer_count = i + 1;
3853                         ret = -EINVAL;
3854                         goto err;
3855                 }
3856                 obj_priv->in_execbuffer = true;
3857         }
3858
3859         /* Move the objects en-masse into the GTT, evicting if necessary. */
3860         ret = i915_gem_execbuffer_pin(dev, file,
3861                                       object_list, exec_list,
3862                                       args->buffer_count);
3863         if (ret)
3864                 goto err;
3865
3866         /* The objects are in their final locations, apply the relocations. */
3867         for (i = 0; i < args->buffer_count; i++) {
3868                 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3869                 obj->base.pending_read_domains = 0;
3870                 obj->base.pending_write_domain = 0;
3871                 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
3872                 if (ret)
3873                         goto err;
3874         }
3875
3876         /* Set the pending read domains for the batch buffer to COMMAND */
3877         batch_obj = object_list[args->buffer_count-1];
3878         if (batch_obj->pending_write_domain) {
3879                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3880                 ret = -EINVAL;
3881                 goto err;
3882         }
3883         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3884
3885         /* Sanity check the batch buffer */
3886         exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3887         ret = i915_gem_check_execbuffer(args, exec_offset);
3888         if (ret != 0) {
3889                 DRM_ERROR("execbuf with invalid offset/length\n");
3890                 goto err;
3891         }
3892
3893         ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
3894                                               object_list, args->buffer_count);
3895         if (ret)
3896                 goto err;
3897
3898 #if WATCH_COHERENCY
3899         for (i = 0; i < args->buffer_count; i++) {
3900                 i915_gem_object_check_coherency(object_list[i],
3901                                                 exec_list[i].handle);
3902         }
3903 #endif
3904
3905 #if WATCH_EXEC
3906         i915_gem_dump_object(batch_obj,
3907                               args->batch_len,
3908                               __func__,
3909                               ~0);
3910 #endif
3911
3912         /* Check for any pending flips. As we only maintain a flip queue depth
3913          * of 1, we can simply insert a WAIT for the next display flip prior
3914          * to executing the batch and avoid stalling the CPU.
3915          */
3916         flips = 0;
3917         for (i = 0; i < args->buffer_count; i++) {
3918                 if (object_list[i]->write_domain)
3919                         flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3920         }
3921         if (flips) {
3922                 int plane, flip_mask;
3923
3924                 for (plane = 0; flips >> plane; plane++) {
3925                         if (((flips >> plane) & 1) == 0)
3926                                 continue;
3927
3928                         if (plane)
3929                                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3930                         else
3931                                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3932
3933                         ret = intel_ring_begin(ring, 2);
3934                         if (ret)
3935                                 goto err;
3936
3937                         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
3938                         intel_ring_emit(ring, MI_NOOP);
3939                         intel_ring_advance(ring);
3940                 }
3941         }
3942
3943         /* Exec the batchbuffer */
3944         ret = ring->dispatch_execbuffer(ring, args, cliprects, exec_offset);
3945         if (ret) {
3946                 DRM_ERROR("dispatch failed %d\n", ret);
3947                 goto err;
3948         }
3949
3950         for (i = 0; i < args->buffer_count; i++) {
3951                 struct drm_gem_object *obj = object_list[i];
3952
3953                 obj->read_domains = obj->pending_read_domains;
3954                 obj->write_domain = obj->pending_write_domain;
3955
3956                 i915_gem_object_move_to_active(obj, ring);
3957                 if (obj->write_domain) {
3958                         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3959                         obj_priv->dirty = 1;
3960                         list_move_tail(&obj_priv->gpu_write_list,
3961                                        &ring->gpu_write_list);
3962                         intel_mark_busy(dev, obj);
3963                 }
3964
3965                 trace_i915_gem_object_change_domain(obj,
3966                                                     obj->read_domains,
3967                                                     obj->write_domain);
3968         }
3969
3970         /*
3971          * Ensure that the commands in the batch buffer are
3972          * finished before the interrupt fires
3973          */
3974         i915_retire_commands(dev, ring);
3975
3976         if (i915_add_request(dev, file, request, ring))
3977                 ring->outstanding_lazy_request = true;
3978         else
3979                 request = NULL;
3980
3981 err:
3982         for (i = 0; i < args->buffer_count; i++) {
3983                 if (object_list[i] == NULL)
3984                     break;
3985
3986                 to_intel_bo(object_list[i])->in_execbuffer = false;
3987                 drm_gem_object_unreference(object_list[i]);
3988         }
3989
3990         mutex_unlock(&dev->struct_mutex);
3991
3992 pre_mutex_err:
3993         drm_free_large(object_list);
3994         kfree(cliprects);
3995         kfree(request);
3996
3997         return ret;
3998 }
3999
4000 /*
4001  * Legacy execbuffer just creates an exec2 list from the original exec object
4002  * list array and passes it to the real function.
4003  */
4004 int
4005 i915_gem_execbuffer(struct drm_device *dev, void *data,
4006                     struct drm_file *file_priv)
4007 {
4008         struct drm_i915_gem_execbuffer *args = data;
4009         struct drm_i915_gem_execbuffer2 exec2;
4010         struct drm_i915_gem_exec_object *exec_list = NULL;
4011         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4012         int ret, i;
4013
4014 #if WATCH_EXEC
4015         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4016                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4017 #endif
4018
4019         if (args->buffer_count < 1) {
4020                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4021                 return -EINVAL;
4022         }
4023
4024         /* Copy in the exec list from userland */
4025         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4026         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4027         if (exec_list == NULL || exec2_list == NULL) {
4028                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4029                           args->buffer_count);
4030                 drm_free_large(exec_list);
4031                 drm_free_large(exec2_list);
4032                 return -ENOMEM;
4033         }
4034         ret = copy_from_user(exec_list,
4035                              (struct drm_i915_relocation_entry __user *)
4036                              (uintptr_t) args->buffers_ptr,
4037                              sizeof(*exec_list) * args->buffer_count);
4038         if (ret != 0) {
4039                 DRM_ERROR("copy %d exec entries failed %d\n",
4040                           args->buffer_count, ret);
4041                 drm_free_large(exec_list);
4042                 drm_free_large(exec2_list);
4043                 return -EFAULT;
4044         }
4045
4046         for (i = 0; i < args->buffer_count; i++) {
4047                 exec2_list[i].handle = exec_list[i].handle;
4048                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4049                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4050                 exec2_list[i].alignment = exec_list[i].alignment;
4051                 exec2_list[i].offset = exec_list[i].offset;
4052                 if (INTEL_INFO(dev)->gen < 4)
4053                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4054                 else
4055                         exec2_list[i].flags = 0;
4056         }
4057
4058         exec2.buffers_ptr = args->buffers_ptr;
4059         exec2.buffer_count = args->buffer_count;
4060         exec2.batch_start_offset = args->batch_start_offset;
4061         exec2.batch_len = args->batch_len;
4062         exec2.DR1 = args->DR1;
4063         exec2.DR4 = args->DR4;
4064         exec2.num_cliprects = args->num_cliprects;
4065         exec2.cliprects_ptr = args->cliprects_ptr;
4066         exec2.flags = I915_EXEC_RENDER;
4067
4068         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4069         if (!ret) {
4070                 /* Copy the new buffer offsets back to the user's exec list. */
4071                 for (i = 0; i < args->buffer_count; i++)
4072                         exec_list[i].offset = exec2_list[i].offset;
4073                 /* ... and back out to userspace */
4074                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4075                                    (uintptr_t) args->buffers_ptr,
4076                                    exec_list,
4077                                    sizeof(*exec_list) * args->buffer_count);
4078                 if (ret) {
4079                         ret = -EFAULT;
4080                         DRM_ERROR("failed to copy %d exec entries "
4081                                   "back to user (%d)\n",
4082                                   args->buffer_count, ret);
4083                 }
4084         }
4085
4086         drm_free_large(exec_list);
4087         drm_free_large(exec2_list);
4088         return ret;
4089 }
4090
4091 int
4092 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4093                      struct drm_file *file_priv)
4094 {
4095         struct drm_i915_gem_execbuffer2 *args = data;
4096         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4097         int ret;
4098
4099 #if WATCH_EXEC
4100         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4101                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4102 #endif
4103
4104         if (args->buffer_count < 1) {
4105                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4106                 return -EINVAL;
4107         }
4108
4109         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4110         if (exec2_list == NULL) {
4111                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4112                           args->buffer_count);
4113                 return -ENOMEM;
4114         }
4115         ret = copy_from_user(exec2_list,
4116                              (struct drm_i915_relocation_entry __user *)
4117                              (uintptr_t) args->buffers_ptr,
4118                              sizeof(*exec2_list) * args->buffer_count);
4119         if (ret != 0) {
4120                 DRM_ERROR("copy %d exec entries failed %d\n",
4121                           args->buffer_count, ret);
4122                 drm_free_large(exec2_list);
4123                 return -EFAULT;
4124         }
4125
4126         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4127         if (!ret) {
4128                 /* Copy the new buffer offsets back to the user's exec list. */
4129                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4130                                    (uintptr_t) args->buffers_ptr,
4131                                    exec2_list,
4132                                    sizeof(*exec2_list) * args->buffer_count);
4133                 if (ret) {
4134                         ret = -EFAULT;
4135                         DRM_ERROR("failed to copy %d exec entries "
4136                                   "back to user (%d)\n",
4137                                   args->buffer_count, ret);
4138                 }
4139         }
4140
4141         drm_free_large(exec2_list);
4142         return ret;
4143 }
4144
4145 int
4146 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment,
4147                     bool mappable, bool need_fence)
4148 {
4149         struct drm_device *dev = obj->dev;
4150         struct drm_i915_private *dev_priv = dev->dev_private;
4151         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4152         int ret;
4153
4154         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4155         WARN_ON(i915_verify_lists(dev));
4156
4157         if (obj_priv->gtt_space != NULL) {
4158                 if ((alignment && obj_priv->gtt_offset & (alignment - 1)) ||
4159                     (need_fence && !obj_priv->fenceable) ||
4160                     (mappable && !obj_priv->mappable)) {
4161                         WARN(obj_priv->pin_count,
4162                              "bo is already pinned with incorrect alignment:"
4163                              " offset=%x, req.alignment=%x, need_fence=%d, fenceable=%d, mappable=%d, cpu_accessible=%d\n",
4164                              obj_priv->gtt_offset, alignment,
4165                              need_fence, obj_priv->fenceable,
4166                              mappable, obj_priv->mappable);
4167                         ret = i915_gem_object_unbind(obj);
4168                         if (ret)
4169                                 return ret;
4170                 }
4171         }
4172
4173         if (obj_priv->gtt_space == NULL) {
4174                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
4175                                                   mappable, need_fence);
4176                 if (ret)
4177                         return ret;
4178         }
4179
4180         if (obj_priv->pin_count++ == 0) {
4181                 i915_gem_info_add_pin(dev_priv, obj_priv, mappable);
4182                 if (!obj_priv->active)
4183                         list_move_tail(&obj_priv->mm_list,
4184                                        &dev_priv->mm.pinned_list);
4185         }
4186         BUG_ON(!obj_priv->pin_mappable && mappable);
4187
4188         WARN_ON(i915_verify_lists(dev));
4189         return 0;
4190 }
4191
4192 void
4193 i915_gem_object_unpin(struct drm_gem_object *obj)
4194 {
4195         struct drm_device *dev = obj->dev;
4196         drm_i915_private_t *dev_priv = dev->dev_private;
4197         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4198
4199         WARN_ON(i915_verify_lists(dev));
4200         BUG_ON(obj_priv->pin_count == 0);
4201         BUG_ON(obj_priv->gtt_space == NULL);
4202
4203         if (--obj_priv->pin_count == 0) {
4204                 if (!obj_priv->active)
4205                         list_move_tail(&obj_priv->mm_list,
4206                                        &dev_priv->mm.inactive_list);
4207                 i915_gem_info_remove_pin(dev_priv, obj_priv);
4208         }
4209         WARN_ON(i915_verify_lists(dev));
4210 }
4211
4212 int
4213 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4214                    struct drm_file *file_priv)
4215 {
4216         struct drm_i915_gem_pin *args = data;
4217         struct drm_gem_object *obj;
4218         struct drm_i915_gem_object *obj_priv;
4219         int ret;
4220
4221         ret = i915_mutex_lock_interruptible(dev);
4222         if (ret)
4223                 return ret;
4224
4225         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4226         if (obj == NULL) {
4227                 ret = -ENOENT;
4228                 goto unlock;
4229         }
4230         obj_priv = to_intel_bo(obj);
4231
4232         if (obj_priv->madv != I915_MADV_WILLNEED) {
4233                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4234                 ret = -EINVAL;
4235                 goto out;
4236         }
4237
4238         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4239                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4240                           args->handle);
4241                 ret = -EINVAL;
4242                 goto out;
4243         }
4244
4245         obj_priv->user_pin_count++;
4246         obj_priv->pin_filp = file_priv;
4247         if (obj_priv->user_pin_count == 1) {
4248                 ret = i915_gem_object_pin(obj, args->alignment,
4249                                           true, obj_priv->tiling_mode);
4250                 if (ret)
4251                         goto out;
4252         }
4253
4254         /* XXX - flush the CPU caches for pinned objects
4255          * as the X server doesn't manage domains yet
4256          */
4257         i915_gem_object_flush_cpu_write_domain(obj);
4258         args->offset = obj_priv->gtt_offset;
4259 out:
4260         drm_gem_object_unreference(obj);
4261 unlock:
4262         mutex_unlock(&dev->struct_mutex);
4263         return ret;
4264 }
4265
4266 int
4267 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4268                      struct drm_file *file_priv)
4269 {
4270         struct drm_i915_gem_pin *args = data;
4271         struct drm_gem_object *obj;
4272         struct drm_i915_gem_object *obj_priv;
4273         int ret;
4274
4275         ret = i915_mutex_lock_interruptible(dev);
4276         if (ret)
4277                 return ret;
4278
4279         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4280         if (obj == NULL) {
4281                 ret = -ENOENT;
4282                 goto unlock;
4283         }
4284         obj_priv = to_intel_bo(obj);
4285
4286         if (obj_priv->pin_filp != file_priv) {
4287                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4288                           args->handle);
4289                 ret = -EINVAL;
4290                 goto out;
4291         }
4292         obj_priv->user_pin_count--;
4293         if (obj_priv->user_pin_count == 0) {
4294                 obj_priv->pin_filp = NULL;
4295                 i915_gem_object_unpin(obj);
4296         }
4297
4298 out:
4299         drm_gem_object_unreference(obj);
4300 unlock:
4301         mutex_unlock(&dev->struct_mutex);
4302         return ret;
4303 }
4304
4305 int
4306 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4307                     struct drm_file *file_priv)
4308 {
4309         struct drm_i915_gem_busy *args = data;
4310         struct drm_gem_object *obj;
4311         struct drm_i915_gem_object *obj_priv;
4312         int ret;
4313
4314         ret = i915_mutex_lock_interruptible(dev);
4315         if (ret)
4316                 return ret;
4317
4318         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4319         if (obj == NULL) {
4320                 ret = -ENOENT;
4321                 goto unlock;
4322         }
4323         obj_priv = to_intel_bo(obj);
4324
4325         /* Count all active objects as busy, even if they are currently not used
4326          * by the gpu. Users of this interface expect objects to eventually
4327          * become non-busy without any further actions, therefore emit any
4328          * necessary flushes here.
4329          */
4330         args->busy = obj_priv->active;
4331         if (args->busy) {
4332                 /* Unconditionally flush objects, even when the gpu still uses this
4333                  * object. Userspace calling this function indicates that it wants to
4334                  * use this buffer rather sooner than later, so issuing the required
4335                  * flush earlier is beneficial.
4336                  */
4337                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4338                         i915_gem_flush_ring(dev, file_priv,
4339                                             obj_priv->ring,
4340                                             0, obj->write_domain);
4341
4342                 /* Update the active list for the hardware's current position.
4343                  * Otherwise this only updates on a delayed timer or when irqs
4344                  * are actually unmasked, and our working set ends up being
4345                  * larger than required.
4346                  */
4347                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4348
4349                 args->busy = obj_priv->active;
4350         }
4351
4352         drm_gem_object_unreference(obj);
4353 unlock:
4354         mutex_unlock(&dev->struct_mutex);
4355         return ret;
4356 }
4357
4358 int
4359 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4360                         struct drm_file *file_priv)
4361 {
4362     return i915_gem_ring_throttle(dev, file_priv);
4363 }
4364
4365 int
4366 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4367                        struct drm_file *file_priv)
4368 {
4369         struct drm_i915_gem_madvise *args = data;
4370         struct drm_gem_object *obj;
4371         struct drm_i915_gem_object *obj_priv;
4372         int ret;
4373
4374         switch (args->madv) {
4375         case I915_MADV_DONTNEED:
4376         case I915_MADV_WILLNEED:
4377             break;
4378         default:
4379             return -EINVAL;
4380         }
4381
4382         ret = i915_mutex_lock_interruptible(dev);
4383         if (ret)
4384                 return ret;
4385
4386         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4387         if (obj == NULL) {
4388                 ret = -ENOENT;
4389                 goto unlock;
4390         }
4391         obj_priv = to_intel_bo(obj);
4392
4393         if (obj_priv->pin_count) {
4394                 ret = -EINVAL;
4395                 goto out;
4396         }
4397
4398         if (obj_priv->madv != __I915_MADV_PURGED)
4399                 obj_priv->madv = args->madv;
4400
4401         /* if the object is no longer bound, discard its backing storage */
4402         if (i915_gem_object_is_purgeable(obj_priv) &&
4403             obj_priv->gtt_space == NULL)
4404                 i915_gem_object_truncate(obj);
4405
4406         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4407
4408 out:
4409         drm_gem_object_unreference(obj);
4410 unlock:
4411         mutex_unlock(&dev->struct_mutex);
4412         return ret;
4413 }
4414
4415 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4416                                               size_t size)
4417 {
4418         struct drm_i915_private *dev_priv = dev->dev_private;
4419         struct drm_i915_gem_object *obj;
4420
4421         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4422         if (obj == NULL)
4423                 return NULL;
4424
4425         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4426                 kfree(obj);
4427                 return NULL;
4428         }
4429
4430         i915_gem_info_add_obj(dev_priv, size);
4431
4432         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4433         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4434
4435         obj->agp_type = AGP_USER_MEMORY;
4436         obj->base.driver_private = NULL;
4437         obj->fence_reg = I915_FENCE_REG_NONE;
4438         INIT_LIST_HEAD(&obj->mm_list);
4439         INIT_LIST_HEAD(&obj->ring_list);
4440         INIT_LIST_HEAD(&obj->gpu_write_list);
4441         obj->madv = I915_MADV_WILLNEED;
4442         obj->fenceable = true;
4443         obj->mappable = true;
4444
4445         return &obj->base;
4446 }
4447
4448 int i915_gem_init_object(struct drm_gem_object *obj)
4449 {
4450         BUG();
4451
4452         return 0;
4453 }
4454
4455 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4456 {
4457         struct drm_device *dev = obj->dev;
4458         drm_i915_private_t *dev_priv = dev->dev_private;
4459         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4460         int ret;
4461
4462         ret = i915_gem_object_unbind(obj);
4463         if (ret == -ERESTARTSYS) {
4464                 list_move(&obj_priv->mm_list,
4465                           &dev_priv->mm.deferred_free_list);
4466                 return;
4467         }
4468
4469         if (obj->map_list.map)
4470                 i915_gem_free_mmap_offset(obj);
4471
4472         drm_gem_object_release(obj);
4473         i915_gem_info_remove_obj(dev_priv, obj->size);
4474
4475         kfree(obj_priv->page_cpu_valid);
4476         kfree(obj_priv->bit_17);
4477         kfree(obj_priv);
4478 }
4479
4480 void i915_gem_free_object(struct drm_gem_object *obj)
4481 {
4482         struct drm_device *dev = obj->dev;
4483         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4484
4485         trace_i915_gem_object_destroy(obj);
4486
4487         while (obj_priv->pin_count > 0)
4488                 i915_gem_object_unpin(obj);
4489
4490         if (obj_priv->phys_obj)
4491                 i915_gem_detach_phys_object(dev, obj);
4492
4493         i915_gem_free_object_tail(obj);
4494 }
4495
4496 int
4497 i915_gem_idle(struct drm_device *dev)
4498 {
4499         drm_i915_private_t *dev_priv = dev->dev_private;
4500         int ret;
4501
4502         mutex_lock(&dev->struct_mutex);
4503
4504         if (dev_priv->mm.suspended) {
4505                 mutex_unlock(&dev->struct_mutex);
4506                 return 0;
4507         }
4508
4509         ret = i915_gpu_idle(dev);
4510         if (ret) {
4511                 mutex_unlock(&dev->struct_mutex);
4512                 return ret;
4513         }
4514
4515         /* Under UMS, be paranoid and evict. */
4516         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4517                 ret = i915_gem_evict_inactive(dev, false);
4518                 if (ret) {
4519                         mutex_unlock(&dev->struct_mutex);
4520                         return ret;
4521                 }
4522         }
4523
4524         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4525          * We need to replace this with a semaphore, or something.
4526          * And not confound mm.suspended!
4527          */
4528         dev_priv->mm.suspended = 1;
4529         del_timer_sync(&dev_priv->hangcheck_timer);
4530
4531         i915_kernel_lost_context(dev);
4532         i915_gem_cleanup_ringbuffer(dev);
4533
4534         mutex_unlock(&dev->struct_mutex);
4535
4536         /* Cancel the retire work handler, which should be idle now. */
4537         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4538
4539         return 0;
4540 }
4541
4542 /*
4543  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4544  * over cache flushing.
4545  */
4546 static int
4547 i915_gem_init_pipe_control(struct drm_device *dev)
4548 {
4549         drm_i915_private_t *dev_priv = dev->dev_private;
4550         struct drm_gem_object *obj;
4551         struct drm_i915_gem_object *obj_priv;
4552         int ret;
4553
4554         obj = i915_gem_alloc_object(dev, 4096);
4555         if (obj == NULL) {
4556                 DRM_ERROR("Failed to allocate seqno page\n");
4557                 ret = -ENOMEM;
4558                 goto err;
4559         }
4560         obj_priv = to_intel_bo(obj);
4561         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4562
4563         ret = i915_gem_object_pin(obj, 4096, true, false);
4564         if (ret)
4565                 goto err_unref;
4566
4567         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4568         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4569         if (dev_priv->seqno_page == NULL)
4570                 goto err_unpin;
4571
4572         dev_priv->seqno_obj = obj;
4573         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4574
4575         return 0;
4576
4577 err_unpin:
4578         i915_gem_object_unpin(obj);
4579 err_unref:
4580         drm_gem_object_unreference(obj);
4581 err:
4582         return ret;
4583 }
4584
4585
4586 static void
4587 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4588 {
4589         drm_i915_private_t *dev_priv = dev->dev_private;
4590         struct drm_gem_object *obj;
4591         struct drm_i915_gem_object *obj_priv;
4592
4593         obj = dev_priv->seqno_obj;
4594         obj_priv = to_intel_bo(obj);
4595         kunmap(obj_priv->pages[0]);
4596         i915_gem_object_unpin(obj);
4597         drm_gem_object_unreference(obj);
4598         dev_priv->seqno_obj = NULL;
4599
4600         dev_priv->seqno_page = NULL;
4601 }
4602
4603 int
4604 i915_gem_init_ringbuffer(struct drm_device *dev)
4605 {
4606         drm_i915_private_t *dev_priv = dev->dev_private;
4607         int ret;
4608
4609         if (HAS_PIPE_CONTROL(dev)) {
4610                 ret = i915_gem_init_pipe_control(dev);
4611                 if (ret)
4612                         return ret;
4613         }
4614
4615         ret = intel_init_render_ring_buffer(dev);
4616         if (ret)
4617                 goto cleanup_pipe_control;
4618
4619         if (HAS_BSD(dev)) {
4620                 ret = intel_init_bsd_ring_buffer(dev);
4621                 if (ret)
4622                         goto cleanup_render_ring;
4623         }
4624
4625         if (HAS_BLT(dev)) {
4626                 ret = intel_init_blt_ring_buffer(dev);
4627                 if (ret)
4628                         goto cleanup_bsd_ring;
4629         }
4630
4631         dev_priv->next_seqno = 1;
4632
4633         return 0;
4634
4635 cleanup_bsd_ring:
4636         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4637 cleanup_render_ring:
4638         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4639 cleanup_pipe_control:
4640         if (HAS_PIPE_CONTROL(dev))
4641                 i915_gem_cleanup_pipe_control(dev);
4642         return ret;
4643 }
4644
4645 void
4646 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4647 {
4648         drm_i915_private_t *dev_priv = dev->dev_private;
4649
4650         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4651         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4652         intel_cleanup_ring_buffer(&dev_priv->blt_ring);
4653         if (HAS_PIPE_CONTROL(dev))
4654                 i915_gem_cleanup_pipe_control(dev);
4655 }
4656
4657 int
4658 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4659                        struct drm_file *file_priv)
4660 {
4661         drm_i915_private_t *dev_priv = dev->dev_private;
4662         int ret;
4663
4664         if (drm_core_check_feature(dev, DRIVER_MODESET))
4665                 return 0;
4666
4667         if (atomic_read(&dev_priv->mm.wedged)) {
4668                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4669                 atomic_set(&dev_priv->mm.wedged, 0);
4670         }
4671
4672         mutex_lock(&dev->struct_mutex);
4673         dev_priv->mm.suspended = 0;
4674
4675         ret = i915_gem_init_ringbuffer(dev);
4676         if (ret != 0) {
4677                 mutex_unlock(&dev->struct_mutex);
4678                 return ret;
4679         }
4680
4681         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4682         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4683         BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4684         BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4685         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4686         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4687         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4688         BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4689         BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4690         mutex_unlock(&dev->struct_mutex);
4691
4692         ret = drm_irq_install(dev);
4693         if (ret)
4694                 goto cleanup_ringbuffer;
4695
4696         return 0;
4697
4698 cleanup_ringbuffer:
4699         mutex_lock(&dev->struct_mutex);
4700         i915_gem_cleanup_ringbuffer(dev);
4701         dev_priv->mm.suspended = 1;
4702         mutex_unlock(&dev->struct_mutex);
4703
4704         return ret;
4705 }
4706
4707 int
4708 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4709                        struct drm_file *file_priv)
4710 {
4711         if (drm_core_check_feature(dev, DRIVER_MODESET))
4712                 return 0;
4713
4714         drm_irq_uninstall(dev);
4715         return i915_gem_idle(dev);
4716 }
4717
4718 void
4719 i915_gem_lastclose(struct drm_device *dev)
4720 {
4721         int ret;
4722
4723         if (drm_core_check_feature(dev, DRIVER_MODESET))
4724                 return;
4725
4726         ret = i915_gem_idle(dev);
4727         if (ret)
4728                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4729 }
4730
4731 static void
4732 init_ring_lists(struct intel_ring_buffer *ring)
4733 {
4734         INIT_LIST_HEAD(&ring->active_list);
4735         INIT_LIST_HEAD(&ring->request_list);
4736         INIT_LIST_HEAD(&ring->gpu_write_list);
4737 }
4738
4739 void
4740 i915_gem_load(struct drm_device *dev)
4741 {
4742         int i;
4743         drm_i915_private_t *dev_priv = dev->dev_private;
4744
4745         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4746         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4747         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4748         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4749         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4750         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4751         init_ring_lists(&dev_priv->render_ring);
4752         init_ring_lists(&dev_priv->bsd_ring);
4753         init_ring_lists(&dev_priv->blt_ring);
4754         for (i = 0; i < 16; i++)
4755                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4756         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4757                           i915_gem_retire_work_handler);
4758         init_completion(&dev_priv->error_completion);
4759
4760         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4761         if (IS_GEN3(dev)) {
4762                 u32 tmp = I915_READ(MI_ARB_STATE);
4763                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4764                         /* arb state is a masked write, so set bit + bit in mask */
4765                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4766                         I915_WRITE(MI_ARB_STATE, tmp);
4767                 }
4768         }
4769
4770         /* Old X drivers will take 0-2 for front, back, depth buffers */
4771         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4772                 dev_priv->fence_reg_start = 3;
4773
4774         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4775                 dev_priv->num_fence_regs = 16;
4776         else
4777                 dev_priv->num_fence_regs = 8;
4778
4779         /* Initialize fence registers to zero */
4780         switch (INTEL_INFO(dev)->gen) {
4781         case 6:
4782                 for (i = 0; i < 16; i++)
4783                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4784                 break;
4785         case 5:
4786         case 4:
4787                 for (i = 0; i < 16; i++)
4788                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4789                 break;
4790         case 3:
4791                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4792                         for (i = 0; i < 8; i++)
4793                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4794         case 2:
4795                 for (i = 0; i < 8; i++)
4796                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4797                 break;
4798         }
4799         i915_gem_detect_bit_6_swizzle(dev);
4800         init_waitqueue_head(&dev_priv->pending_flip_queue);
4801
4802         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4803         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4804         register_shrinker(&dev_priv->mm.inactive_shrinker);
4805 }
4806
4807 /*
4808  * Create a physically contiguous memory object for this object
4809  * e.g. for cursor + overlay regs
4810  */
4811 static int i915_gem_init_phys_object(struct drm_device *dev,
4812                                      int id, int size, int align)
4813 {
4814         drm_i915_private_t *dev_priv = dev->dev_private;
4815         struct drm_i915_gem_phys_object *phys_obj;
4816         int ret;
4817
4818         if (dev_priv->mm.phys_objs[id - 1] || !size)
4819                 return 0;
4820
4821         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4822         if (!phys_obj)
4823                 return -ENOMEM;
4824
4825         phys_obj->id = id;
4826
4827         phys_obj->handle = drm_pci_alloc(dev, size, align);
4828         if (!phys_obj->handle) {
4829                 ret = -ENOMEM;
4830                 goto kfree_obj;
4831         }
4832 #ifdef CONFIG_X86
4833         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4834 #endif
4835
4836         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4837
4838         return 0;
4839 kfree_obj:
4840         kfree(phys_obj);
4841         return ret;
4842 }
4843
4844 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4845 {
4846         drm_i915_private_t *dev_priv = dev->dev_private;
4847         struct drm_i915_gem_phys_object *phys_obj;
4848
4849         if (!dev_priv->mm.phys_objs[id - 1])
4850                 return;
4851
4852         phys_obj = dev_priv->mm.phys_objs[id - 1];
4853         if (phys_obj->cur_obj) {
4854                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4855         }
4856
4857 #ifdef CONFIG_X86
4858         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4859 #endif
4860         drm_pci_free(dev, phys_obj->handle);
4861         kfree(phys_obj);
4862         dev_priv->mm.phys_objs[id - 1] = NULL;
4863 }
4864
4865 void i915_gem_free_all_phys_object(struct drm_device *dev)
4866 {
4867         int i;
4868
4869         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4870                 i915_gem_free_phys_object(dev, i);
4871 }
4872
4873 void i915_gem_detach_phys_object(struct drm_device *dev,
4874                                  struct drm_gem_object *obj)
4875 {
4876         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4877         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4878         char *vaddr;
4879         int i;
4880         int page_count;
4881
4882         if (!obj_priv->phys_obj)
4883                 return;
4884         vaddr = obj_priv->phys_obj->handle->vaddr;
4885
4886         page_count = obj->size / PAGE_SIZE;
4887
4888         for (i = 0; i < page_count; i++) {
4889                 struct page *page = read_cache_page_gfp(mapping, i,
4890                                                         GFP_HIGHUSER | __GFP_RECLAIMABLE);
4891                 if (!IS_ERR(page)) {
4892                         char *dst = kmap_atomic(page);
4893                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4894                         kunmap_atomic(dst);
4895
4896                         drm_clflush_pages(&page, 1);
4897
4898                         set_page_dirty(page);
4899                         mark_page_accessed(page);
4900                         page_cache_release(page);
4901                 }
4902         }
4903         drm_agp_chipset_flush(dev);
4904
4905         obj_priv->phys_obj->cur_obj = NULL;
4906         obj_priv->phys_obj = NULL;
4907 }
4908
4909 int
4910 i915_gem_attach_phys_object(struct drm_device *dev,
4911                             struct drm_gem_object *obj,
4912                             int id,
4913                             int align)
4914 {
4915         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4916         drm_i915_private_t *dev_priv = dev->dev_private;
4917         struct drm_i915_gem_object *obj_priv;
4918         int ret = 0;
4919         int page_count;
4920         int i;
4921
4922         if (id > I915_MAX_PHYS_OBJECT)
4923                 return -EINVAL;
4924
4925         obj_priv = to_intel_bo(obj);
4926
4927         if (obj_priv->phys_obj) {
4928                 if (obj_priv->phys_obj->id == id)
4929                         return 0;
4930                 i915_gem_detach_phys_object(dev, obj);
4931         }
4932
4933         /* create a new object */
4934         if (!dev_priv->mm.phys_objs[id - 1]) {
4935                 ret = i915_gem_init_phys_object(dev, id,
4936                                                 obj->size, align);
4937                 if (ret) {
4938                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4939                         return ret;
4940                 }
4941         }
4942
4943         /* bind to the object */
4944         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4945         obj_priv->phys_obj->cur_obj = obj;
4946
4947         page_count = obj->size / PAGE_SIZE;
4948
4949         for (i = 0; i < page_count; i++) {
4950                 struct page *page;
4951                 char *dst, *src;
4952
4953                 page = read_cache_page_gfp(mapping, i,
4954                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
4955                 if (IS_ERR(page))
4956                         return PTR_ERR(page);
4957
4958                 src = kmap_atomic(page);
4959                 dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4960                 memcpy(dst, src, PAGE_SIZE);
4961                 kunmap_atomic(src);
4962
4963                 mark_page_accessed(page);
4964                 page_cache_release(page);
4965         }
4966
4967         return 0;
4968 }
4969
4970 static int
4971 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4972                      struct drm_i915_gem_pwrite *args,
4973                      struct drm_file *file_priv)
4974 {
4975         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4976         void *obj_addr;
4977         int ret;
4978         char __user *user_data;
4979
4980         user_data = (char __user *) (uintptr_t) args->data_ptr;
4981         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4982
4983         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4984         ret = copy_from_user(obj_addr, user_data, args->size);
4985         if (ret)
4986                 return -EFAULT;
4987
4988         drm_agp_chipset_flush(dev);
4989         return 0;
4990 }
4991
4992 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4993 {
4994         struct drm_i915_file_private *file_priv = file->driver_priv;
4995
4996         /* Clean up our request list when the client is going away, so that
4997          * later retire_requests won't dereference our soon-to-be-gone
4998          * file_priv.
4999          */
5000         spin_lock(&file_priv->mm.lock);
5001         while (!list_empty(&file_priv->mm.request_list)) {
5002                 struct drm_i915_gem_request *request;
5003
5004                 request = list_first_entry(&file_priv->mm.request_list,
5005                                            struct drm_i915_gem_request,
5006                                            client_list);
5007                 list_del(&request->client_list);
5008                 request->file_priv = NULL;
5009         }
5010         spin_unlock(&file_priv->mm.lock);
5011 }
5012
5013 static int
5014 i915_gpu_is_active(struct drm_device *dev)
5015 {
5016         drm_i915_private_t *dev_priv = dev->dev_private;
5017         int lists_empty;
5018
5019         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5020                       list_empty(&dev_priv->mm.active_list);
5021
5022         return !lists_empty;
5023 }
5024
5025 static int
5026 i915_gem_inactive_shrink(struct shrinker *shrinker,
5027                          int nr_to_scan,
5028                          gfp_t gfp_mask)
5029 {
5030         struct drm_i915_private *dev_priv =
5031                 container_of(shrinker,
5032                              struct drm_i915_private,
5033                              mm.inactive_shrinker);
5034         struct drm_device *dev = dev_priv->dev;
5035         struct drm_i915_gem_object *obj, *next;
5036         int cnt;
5037
5038         if (!mutex_trylock(&dev->struct_mutex))
5039                 return 0;
5040
5041         /* "fast-path" to count number of available objects */
5042         if (nr_to_scan == 0) {
5043                 cnt = 0;
5044                 list_for_each_entry(obj,
5045                                     &dev_priv->mm.inactive_list,
5046                                     mm_list)
5047                         cnt++;
5048                 mutex_unlock(&dev->struct_mutex);
5049                 return cnt / 100 * sysctl_vfs_cache_pressure;
5050         }
5051
5052 rescan:
5053         /* first scan for clean buffers */
5054         i915_gem_retire_requests(dev);
5055
5056         list_for_each_entry_safe(obj, next,
5057                                  &dev_priv->mm.inactive_list,
5058                                  mm_list) {
5059                 if (i915_gem_object_is_purgeable(obj)) {
5060                         i915_gem_object_unbind(&obj->base);
5061                         if (--nr_to_scan == 0)
5062                                 break;
5063                 }
5064         }
5065
5066         /* second pass, evict/count anything still on the inactive list */
5067         cnt = 0;
5068         list_for_each_entry_safe(obj, next,
5069                                  &dev_priv->mm.inactive_list,
5070                                  mm_list) {
5071                 if (nr_to_scan) {
5072                         i915_gem_object_unbind(&obj->base);
5073                         nr_to_scan--;
5074                 } else
5075                         cnt++;
5076         }
5077
5078         if (nr_to_scan && i915_gpu_is_active(dev)) {
5079                 /*
5080                  * We are desperate for pages, so as a last resort, wait
5081                  * for the GPU to finish and discard whatever we can.
5082                  * This has a dramatic impact to reduce the number of
5083                  * OOM-killer events whilst running the GPU aggressively.
5084                  */
5085                 if (i915_gpu_idle(dev) == 0)
5086                         goto rescan;
5087         }
5088         mutex_unlock(&dev->struct_mutex);
5089         return cnt / 100 * sysctl_vfs_cache_pressure;
5090 }