drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv);
  40 static uint32_t i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv);
  41
  42 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
  43                                                   bool pipelined);
  44 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  45 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  46 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  47                                              int write);
  48 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  49                                                      uint64_t offset,
  50                                                      uint64_t size);
  51 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  52 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  53                                           bool interruptible);
  54 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  55                                        unsigned alignment,
  56                                        bool mappable,
  57                                        bool need_fence);
  58 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  59 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  60                                 struct drm_i915_gem_pwrite *args,
  61                                 struct drm_file *file_priv);
  62 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  63
  64 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  65                                     int nr_to_scan,
  66                                     gfp_t gfp_mask);
  67
  68
  69 /* some bookkeeping */
  70 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  71                                   size_t size)
  72 {
  73         dev_priv->mm.object_count++;
  74         dev_priv->mm.object_memory += size;
  75 }
  76
  77 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  78                                      size_t size)
  79 {
  80         dev_priv->mm.object_count--;
  81         dev_priv->mm.object_memory -= size;
  82 }
  83
  84 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
  85                                   struct drm_i915_gem_object *obj)
  86 {
  87         dev_priv->mm.gtt_count++;
  88         dev_priv->mm.gtt_memory += obj->gtt_space->size;
  89         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
  90                 dev_priv->mm.mappable_gtt_used +=
  91                         min_t(size_t, obj->gtt_space->size,
  92                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
  93         }
  94 }
  95
  96 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
  97                                      struct drm_i915_gem_object *obj)
  98 {
  99         dev_priv->mm.gtt_count--;
 100         dev_priv->mm.gtt_memory -= obj->gtt_space->size;
 101         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
 102                 dev_priv->mm.mappable_gtt_used -=
 103                         min_t(size_t, obj->gtt_space->size,
 104                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
 105         }
 106 }
 107
 108 /**
 109  * Update the mappable working set counters. Call _only_ when there is a change
 110  * in one of (pin|fault)_mappable and update *_mappable _before_ calling.
 111  * @mappable: new state the changed mappable flag (either pin_ or fault_).
 112  */
 113 static void
 114 i915_gem_info_update_mappable(struct drm_i915_private *dev_priv,
 115                               struct drm_i915_gem_object *obj,
 116                               bool mappable)
 117 {
 118         if (mappable) {
 119                 if (obj->pin_mappable && obj->fault_mappable)
 120                         /* Combined state was already mappable. */
 121                         return;
 122                 dev_priv->mm.gtt_mappable_count++;
 123                 dev_priv->mm.gtt_mappable_memory += obj->gtt_space->size;
 124         } else {
 125                 if (obj->pin_mappable || obj->fault_mappable)
 126                         /* Combined state still mappable. */
 127                         return;
 128                 dev_priv->mm.gtt_mappable_count--;
 129                 dev_priv->mm.gtt_mappable_memory -= obj->gtt_space->size;
 130         }
 131 }
 132
 133 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
 134                                   struct drm_i915_gem_object *obj,
 135                                   bool mappable)
 136 {
 137         dev_priv->mm.pin_count++;
 138         dev_priv->mm.pin_memory += obj->gtt_space->size;
 139         if (mappable) {
 140                 obj->pin_mappable = true;
 141                 i915_gem_info_update_mappable(dev_priv, obj, true);
 142         }
 143 }
 144
 145 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
 146                                      struct drm_i915_gem_object *obj)
 147 {
 148         dev_priv->mm.pin_count--;
 149         dev_priv->mm.pin_memory -= obj->gtt_space->size;
 150         if (obj->pin_mappable) {
 151                 obj->pin_mappable = false;
 152                 i915_gem_info_update_mappable(dev_priv, obj, false);
 153         }
 154 }
 155
 156 int
 157 i915_gem_check_is_wedged(struct drm_device *dev)
 158 {
 159         struct drm_i915_private *dev_priv = dev->dev_private;
 160         struct completion *x = &dev_priv->error_completion;
 161         unsigned long flags;
 162         int ret;
 163
 164         if (!atomic_read(&dev_priv->mm.wedged))
 165                 return 0;
 166
 167         ret = wait_for_completion_interruptible(x);
 168         if (ret)
 169                 return ret;
 170
 171         /* Success, we reset the GPU! */
 172         if (!atomic_read(&dev_priv->mm.wedged))
 173                 return 0;
 174
 175         /* GPU is hung, bump the completion count to account for
 176          * the token we just consumed so that we never hit zero and
 177          * end up waiting upon a subsequent completion event that
 178          * will never happen.
 179          */
 180         spin_lock_irqsave(&x->wait.lock, flags);
 181         x->done++;
 182         spin_unlock_irqrestore(&x->wait.lock, flags);
 183         return -EIO;
 184 }
 185
 186 static int i915_mutex_lock_interruptible(struct drm_device *dev)
 187 {
 188         struct drm_i915_private *dev_priv = dev->dev_private;
 189         int ret;
 190
 191         ret = i915_gem_check_is_wedged(dev);
 192         if (ret)
 193                 return ret;
 194
 195         ret = mutex_lock_interruptible(&dev->struct_mutex);
 196         if (ret)
 197                 return ret;
 198
 199         if (atomic_read(&dev_priv->mm.wedged)) {
 200                 mutex_unlock(&dev->struct_mutex);
 201                 return -EAGAIN;
 202         }
 203
 204         WARN_ON(i915_verify_lists(dev));
 205         return 0;
 206 }
 207
 208 static inline bool
 209 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
 210 {
 211         return obj_priv->gtt_space &&
 212                 !obj_priv->active &&
 213                 obj_priv->pin_count == 0;
 214 }
 215
 216 int i915_gem_do_init(struct drm_device *dev,
 217                      unsigned long start,
 218                      unsigned long mappable_end,
 219                      unsigned long end)
 220 {
 221         drm_i915_private_t *dev_priv = dev->dev_private;
 222
 223         if (start >= end ||
 224             (start & (PAGE_SIZE - 1)) != 0 ||
 225             (end & (PAGE_SIZE - 1)) != 0) {
 226                 return -EINVAL;
 227         }
 228
 229         drm_mm_init(&dev_priv->mm.gtt_space, start,
 230                     end - start);
 231
 232         dev_priv->mm.gtt_total = end - start;
 233         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 234         dev_priv->mm.gtt_mappable_end = mappable_end;
 235
 236         return 0;
 237 }
 238
 239 int
 240 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 241                     struct drm_file *file_priv)
 242 {
 243         struct drm_i915_gem_init *args = data;
 244         int ret;
 245
 246         mutex_lock(&dev->struct_mutex);
 247         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 248         mutex_unlock(&dev->struct_mutex);
 249
 250         return ret;
 251 }
 252
 253 int
 254 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 255                             struct drm_file *file_priv)
 256 {
 257         struct drm_i915_private *dev_priv = dev->dev_private;
 258         struct drm_i915_gem_get_aperture *args = data;
 259
 260         if (!(dev->driver->driver_features & DRIVER_GEM))
 261                 return -ENODEV;
 262
 263         mutex_lock(&dev->struct_mutex);
 264         args->aper_size = dev_priv->mm.gtt_total;
 265         args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
 266         mutex_unlock(&dev->struct_mutex);
 267
 268         return 0;
 269 }
 270
 271
 272 /**
 273  * Creates a new mm object and returns a handle to it.
 274  */
 275 int
 276 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 277                       struct drm_file *file_priv)
 278 {
 279         struct drm_i915_gem_create *args = data;
 280         struct drm_gem_object *obj;
 281         int ret;
 282         u32 handle;
 283
 284         args->size = roundup(args->size, PAGE_SIZE);
 285
 286         /* Allocate the new object */
 287         obj = i915_gem_alloc_object(dev, args->size);
 288         if (obj == NULL)
 289                 return -ENOMEM;
 290
 291         ret = drm_gem_handle_create(file_priv, obj, &handle);
 292         if (ret) {
 293                 drm_gem_object_release(obj);
 294                 i915_gem_info_remove_obj(dev->dev_private, obj->size);
 295                 kfree(obj);
 296                 return ret;
 297         }
 298
 299         /* drop reference from allocate - handle holds it now */
 300         drm_gem_object_unreference(obj);
 301         trace_i915_gem_object_create(obj);
 302
 303         args->handle = handle;
 304         return 0;
 305 }
 306
 307 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 308 {
 309         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 310         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 311
 312         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 313                 obj_priv->tiling_mode != I915_TILING_NONE;
 314 }
 315
 316 static inline void
 317 slow_shmem_copy(struct page *dst_page,
 318                 int dst_offset,
 319                 struct page *src_page,
 320                 int src_offset,
 321                 int length)
 322 {
 323         char *dst_vaddr, *src_vaddr;
 324
 325         dst_vaddr = kmap(dst_page);
 326         src_vaddr = kmap(src_page);
 327
 328         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 329
 330         kunmap(src_page);
 331         kunmap(dst_page);
 332 }
 333
 334 static inline void
 335 slow_shmem_bit17_copy(struct page *gpu_page,
 336                       int gpu_offset,
 337                       struct page *cpu_page,
 338                       int cpu_offset,
 339                       int length,
 340                       int is_read)
 341 {
 342         char *gpu_vaddr, *cpu_vaddr;
 343
 344         /* Use the unswizzled path if this page isn't affected. */
 345         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 346                 if (is_read)
 347                         return slow_shmem_copy(cpu_page, cpu_offset,
 348                                                gpu_page, gpu_offset, length);
 349                 else
 350                         return slow_shmem_copy(gpu_page, gpu_offset,
 351                                                cpu_page, cpu_offset, length);
 352         }
 353
 354         gpu_vaddr = kmap(gpu_page);
 355         cpu_vaddr = kmap(cpu_page);
 356
 357         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 358          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 359          */
 360         while (length > 0) {
 361                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 362                 int this_length = min(cacheline_end - gpu_offset, length);
 363                 int swizzled_gpu_offset = gpu_offset ^ 64;
 364
 365                 if (is_read) {
 366                         memcpy(cpu_vaddr + cpu_offset,
 367                                gpu_vaddr + swizzled_gpu_offset,
 368                                this_length);
 369                 } else {
 370                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 371                                cpu_vaddr + cpu_offset,
 372                                this_length);
 373                 }
 374                 cpu_offset += this_length;
 375                 gpu_offset += this_length;
 376                 length -= this_length;
 377         }
 378
 379         kunmap(cpu_page);
 380         kunmap(gpu_page);
 381 }
 382
 383 /**
 384  * This is the fast shmem pread path, which attempts to copy_from_user directly
 385  * from the backing pages of the object to the user's address space.  On a
 386  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 387  */
 388 static int
 389 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 390                           struct drm_i915_gem_pread *args,
 391                           struct drm_file *file_priv)
 392 {
 393         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 394         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 395         ssize_t remain;
 396         loff_t offset;
 397         char __user *user_data;
 398         int page_offset, page_length;
 399
 400         user_data = (char __user *) (uintptr_t) args->data_ptr;
 401         remain = args->size;
 402
 403         obj_priv = to_intel_bo(obj);
 404         offset = args->offset;
 405
 406         while (remain > 0) {
 407                 struct page *page;
 408                 char *vaddr;
 409                 int ret;
 410
 411                 /* Operation in this page
 412                  *
 413                  * page_offset = offset within page
 414                  * page_length = bytes to copy for this page
 415                  */
 416                 page_offset = offset & (PAGE_SIZE-1);
 417                 page_length = remain;
 418                 if ((page_offset + remain) > PAGE_SIZE)
 419                         page_length = PAGE_SIZE - page_offset;
 420
 421                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 422                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 423                 if (IS_ERR(page))
 424                         return PTR_ERR(page);
 425
 426                 vaddr = kmap_atomic(page);
 427                 ret = __copy_to_user_inatomic(user_data,
 428                                               vaddr + page_offset,
 429                                               page_length);
 430                 kunmap_atomic(vaddr);
 431
 432                 mark_page_accessed(page);
 433                 page_cache_release(page);
 434                 if (ret)
 435                         return -EFAULT;
 436
 437                 remain -= page_length;
 438                 user_data += page_length;
 439                 offset += page_length;
 440         }
 441
 442         return 0;
 443 }
 444
 445 /**
 446  * This is the fallback shmem pread path, which allocates temporary storage
 447  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 448  * can copy out of the object's backing pages while holding the struct mutex
 449  * and not take page faults.
 450  */
 451 static int
 452 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 453                           struct drm_i915_gem_pread *args,
 454                           struct drm_file *file_priv)
 455 {
 456         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 457         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 458         struct mm_struct *mm = current->mm;
 459         struct page **user_pages;
 460         ssize_t remain;
 461         loff_t offset, pinned_pages, i;
 462         loff_t first_data_page, last_data_page, num_pages;
 463         int shmem_page_offset;
 464         int data_page_index, data_page_offset;
 465         int page_length;
 466         int ret;
 467         uint64_t data_ptr = args->data_ptr;
 468         int do_bit17_swizzling;
 469
 470         remain = args->size;
 471
 472         /* Pin the user pages containing the data.  We can't fault while
 473          * holding the struct mutex, yet we want to hold it while
 474          * dereferencing the user data.
 475          */
 476         first_data_page = data_ptr / PAGE_SIZE;
 477         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 478         num_pages = last_data_page - first_data_page + 1;
 479
 480         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 481         if (user_pages == NULL)
 482                 return -ENOMEM;
 483
 484         mutex_unlock(&dev->struct_mutex);
 485         down_read(&mm->mmap_sem);
 486         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 487                                       num_pages, 1, 0, user_pages, NULL);
 488         up_read(&mm->mmap_sem);
 489         mutex_lock(&dev->struct_mutex);
 490         if (pinned_pages < num_pages) {
 491                 ret = -EFAULT;
 492                 goto out;
 493         }
 494
 495         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 496                                                         args->offset,
 497                                                         args->size);
 498         if (ret)
 499                 goto out;
 500
 501         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 502
 503         obj_priv = to_intel_bo(obj);
 504         offset = args->offset;
 505
 506         while (remain > 0) {
 507                 struct page *page;
 508
 509                 /* Operation in this page
 510                  *
 511                  * shmem_page_offset = offset within page in shmem file
 512                  * data_page_index = page number in get_user_pages return
 513                  * data_page_offset = offset with data_page_index page.
 514                  * page_length = bytes to copy for this page
 515                  */
 516                 shmem_page_offset = offset & ~PAGE_MASK;
 517                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 518                 data_page_offset = data_ptr & ~PAGE_MASK;
 519
 520                 page_length = remain;
 521                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 522                         page_length = PAGE_SIZE - shmem_page_offset;
 523                 if ((data_page_offset + page_length) > PAGE_SIZE)
 524                         page_length = PAGE_SIZE - data_page_offset;
 525
 526                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 527                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 528                 if (IS_ERR(page))
 529                         return PTR_ERR(page);
 530
 531                 if (do_bit17_swizzling) {
 532                         slow_shmem_bit17_copy(page,
 533                                               shmem_page_offset,
 534                                               user_pages[data_page_index],
 535                                               data_page_offset,
 536                                               page_length,
 537                                               1);
 538                 } else {
 539                         slow_shmem_copy(user_pages[data_page_index],
 540                                         data_page_offset,
 541                                         page,
 542                                         shmem_page_offset,
 543                                         page_length);
 544                 }
 545
 546                 mark_page_accessed(page);
 547                 page_cache_release(page);
 548
 549                 remain -= page_length;
 550                 data_ptr += page_length;
 551                 offset += page_length;
 552         }
 553
 554 out:
 555         for (i = 0; i < pinned_pages; i++) {
 556                 SetPageDirty(user_pages[i]);
 557                 mark_page_accessed(user_pages[i]);
 558                 page_cache_release(user_pages[i]);
 559         }
 560         drm_free_large(user_pages);
 561
 562         return ret;
 563 }
 564
 565 /**
 566  * Reads data from the object referenced by handle.
 567  *
 568  * On error, the contents of *data are undefined.
 569  */
 570 int
 571 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 572                      struct drm_file *file_priv)
 573 {
 574         struct drm_i915_gem_pread *args = data;
 575         struct drm_gem_object *obj;
 576         struct drm_i915_gem_object *obj_priv;
 577         int ret = 0;
 578
 579         ret = i915_mutex_lock_interruptible(dev);
 580         if (ret)
 581                 return ret;
 582
 583         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 584         if (obj == NULL) {
 585                 ret = -ENOENT;
 586                 goto unlock;
 587         }
 588         obj_priv = to_intel_bo(obj);
 589
 590         /* Bounds check source.  */
 591         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 592                 ret = -EINVAL;
 593                 goto out;
 594         }
 595
 596         if (args->size == 0)
 597                 goto out;
 598
 599         if (!access_ok(VERIFY_WRITE,
 600                        (char __user *)(uintptr_t)args->data_ptr,
 601                        args->size)) {
 602                 ret = -EFAULT;
 603                 goto out;
 604         }
 605
 606         ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
 607                                        args->size);
 608         if (ret) {
 609                 ret = -EFAULT;
 610                 goto out;
 611         }
 612
 613         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 614                                                         args->offset,
 615                                                         args->size);
 616         if (ret)
 617                 goto out;
 618
 619         ret = -EFAULT;
 620         if (!i915_gem_object_needs_bit17_swizzle(obj))
 621                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 622         if (ret == -EFAULT)
 623                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 624
 625 out:
 626         drm_gem_object_unreference(obj);
 627 unlock:
 628         mutex_unlock(&dev->struct_mutex);
 629         return ret;
 630 }
 631
 632 /* This is the fast write path which cannot handle
 633  * page faults in the source data
 634  */
 635
 636 static inline int
 637 fast_user_write(struct io_mapping *mapping,
 638                 loff_t page_base, int page_offset,
 639                 char __user *user_data,
 640                 int length)
 641 {
 642         char *vaddr_atomic;
 643         unsigned long unwritten;
 644
 645         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 646         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 647                                                       user_data, length);
 648         io_mapping_unmap_atomic(vaddr_atomic);
 649         return unwritten;
 650 }
 651
 652 /* Here's the write path which can sleep for
 653  * page faults
 654  */
 655
 656 static inline void
 657 slow_kernel_write(struct io_mapping *mapping,
 658                   loff_t gtt_base, int gtt_offset,
 659                   struct page *user_page, int user_offset,
 660                   int length)
 661 {
 662         char __iomem *dst_vaddr;
 663         char *src_vaddr;
 664
 665         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 666         src_vaddr = kmap(user_page);
 667
 668         memcpy_toio(dst_vaddr + gtt_offset,
 669                     src_vaddr + user_offset,
 670                     length);
 671
 672         kunmap(user_page);
 673         io_mapping_unmap(dst_vaddr);
 674 }
 675
 676 /**
 677  * This is the fast pwrite path, where we copy the data directly from the
 678  * user into the GTT, uncached.
 679  */
 680 static int
 681 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 682                          struct drm_i915_gem_pwrite *args,
 683                          struct drm_file *file_priv)
 684 {
 685         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 686         drm_i915_private_t *dev_priv = dev->dev_private;
 687         ssize_t remain;
 688         loff_t offset, page_base;
 689         char __user *user_data;
 690         int page_offset, page_length;
 691
 692         user_data = (char __user *) (uintptr_t) args->data_ptr;
 693         remain = args->size;
 694
 695         obj_priv = to_intel_bo(obj);
 696         offset = obj_priv->gtt_offset + args->offset;
 697
 698         while (remain > 0) {
 699                 /* Operation in this page
 700                  *
 701                  * page_base = page offset within aperture
 702                  * page_offset = offset within page
 703                  * page_length = bytes to copy for this page
 704                  */
 705                 page_base = (offset & ~(PAGE_SIZE-1));
 706                 page_offset = offset & (PAGE_SIZE-1);
 707                 page_length = remain;
 708                 if ((page_offset + remain) > PAGE_SIZE)
 709                         page_length = PAGE_SIZE - page_offset;
 710
 711                 /* If we get a fault while copying data, then (presumably) our
 712                  * source page isn't available.  Return the error and we'll
 713                  * retry in the slow path.
 714                  */
 715                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 716                                     page_offset, user_data, page_length))
 717
 718                         return -EFAULT;
 719
 720                 remain -= page_length;
 721                 user_data += page_length;
 722                 offset += page_length;
 723         }
 724
 725         return 0;
 726 }
 727
 728 /**
 729  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 730  * the memory and maps it using kmap_atomic for copying.
 731  *
 732  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 733  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 734  */
 735 static int
 736 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 737                          struct drm_i915_gem_pwrite *args,
 738                          struct drm_file *file_priv)
 739 {
 740         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 741         drm_i915_private_t *dev_priv = dev->dev_private;
 742         ssize_t remain;
 743         loff_t gtt_page_base, offset;
 744         loff_t first_data_page, last_data_page, num_pages;
 745         loff_t pinned_pages, i;
 746         struct page **user_pages;
 747         struct mm_struct *mm = current->mm;
 748         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 749         int ret;
 750         uint64_t data_ptr = args->data_ptr;
 751
 752         remain = args->size;
 753
 754         /* Pin the user pages containing the data.  We can't fault while
 755          * holding the struct mutex, and all of the pwrite implementations
 756          * want to hold it while dereferencing the user data.
 757          */
 758         first_data_page = data_ptr / PAGE_SIZE;
 759         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 760         num_pages = last_data_page - first_data_page + 1;
 761
 762         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 763         if (user_pages == NULL)
 764                 return -ENOMEM;
 765
 766         mutex_unlock(&dev->struct_mutex);
 767         down_read(&mm->mmap_sem);
 768         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 769                                       num_pages, 0, 0, user_pages, NULL);
 770         up_read(&mm->mmap_sem);
 771         mutex_lock(&dev->struct_mutex);
 772         if (pinned_pages < num_pages) {
 773                 ret = -EFAULT;
 774                 goto out_unpin_pages;
 775         }
 776
 777         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 778         if (ret)
 779                 goto out_unpin_pages;
 780
 781         obj_priv = to_intel_bo(obj);
 782         offset = obj_priv->gtt_offset + args->offset;
 783
 784         while (remain > 0) {
 785                 /* Operation in this page
 786                  *
 787                  * gtt_page_base = page offset within aperture
 788                  * gtt_page_offset = offset within page in aperture
 789                  * data_page_index = page number in get_user_pages return
 790                  * data_page_offset = offset with data_page_index page.
 791                  * page_length = bytes to copy for this page
 792                  */
 793                 gtt_page_base = offset & PAGE_MASK;
 794                 gtt_page_offset = offset & ~PAGE_MASK;
 795                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 796                 data_page_offset = data_ptr & ~PAGE_MASK;
 797
 798                 page_length = remain;
 799                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 800                         page_length = PAGE_SIZE - gtt_page_offset;
 801                 if ((data_page_offset + page_length) > PAGE_SIZE)
 802                         page_length = PAGE_SIZE - data_page_offset;
 803
 804                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 805                                   gtt_page_base, gtt_page_offset,
 806                                   user_pages[data_page_index],
 807                                   data_page_offset,
 808                                   page_length);
 809
 810                 remain -= page_length;
 811                 offset += page_length;
 812                 data_ptr += page_length;
 813         }
 814
 815 out_unpin_pages:
 816         for (i = 0; i < pinned_pages; i++)
 817                 page_cache_release(user_pages[i]);
 818         drm_free_large(user_pages);
 819
 820         return ret;
 821 }
 822
 823 /**
 824  * This is the fast shmem pwrite path, which attempts to directly
 825  * copy_from_user into the kmapped pages backing the object.
 826  */
 827 static int
 828 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 829                            struct drm_i915_gem_pwrite *args,
 830                            struct drm_file *file_priv)
 831 {
 832         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 833         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 834         ssize_t remain;
 835         loff_t offset;
 836         char __user *user_data;
 837         int page_offset, page_length;
 838
 839         user_data = (char __user *) (uintptr_t) args->data_ptr;
 840         remain = args->size;
 841
 842         obj_priv = to_intel_bo(obj);
 843         offset = args->offset;
 844         obj_priv->dirty = 1;
 845
 846         while (remain > 0) {
 847                 struct page *page;
 848                 char *vaddr;
 849                 int ret;
 850
 851                 /* Operation in this page
 852                  *
 853                  * page_offset = offset within page
 854                  * page_length = bytes to copy for this page
 855                  */
 856                 page_offset = offset & (PAGE_SIZE-1);
 857                 page_length = remain;
 858                 if ((page_offset + remain) > PAGE_SIZE)
 859                         page_length = PAGE_SIZE - page_offset;
 860
 861                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 862                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 863                 if (IS_ERR(page))
 864                         return PTR_ERR(page);
 865
 866                 vaddr = kmap_atomic(page, KM_USER0);
 867                 ret = __copy_from_user_inatomic(vaddr + page_offset,
 868                                                 user_data,
 869                                                 page_length);
 870                 kunmap_atomic(vaddr, KM_USER0);
 871
 872                 set_page_dirty(page);
 873                 mark_page_accessed(page);
 874                 page_cache_release(page);
 875
 876                 /* If we get a fault while copying data, then (presumably) our
 877                  * source page isn't available.  Return the error and we'll
 878                  * retry in the slow path.
 879                  */
 880                 if (ret)
 881                         return -EFAULT;
 882
 883                 remain -= page_length;
 884                 user_data += page_length;
 885                 offset += page_length;
 886         }
 887
 888         return 0;
 889 }
 890
 891 /**
 892  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 893  * the memory and maps it using kmap_atomic for copying.
 894  *
 895  * This avoids taking mmap_sem for faulting on the user's address while the
 896  * struct_mutex is held.
 897  */
 898 static int
 899 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 900                            struct drm_i915_gem_pwrite *args,
 901                            struct drm_file *file_priv)
 902 {
 903         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 904         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 905         struct mm_struct *mm = current->mm;
 906         struct page **user_pages;
 907         ssize_t remain;
 908         loff_t offset, pinned_pages, i;
 909         loff_t first_data_page, last_data_page, num_pages;
 910         int shmem_page_offset;
 911         int data_page_index,  data_page_offset;
 912         int page_length;
 913         int ret;
 914         uint64_t data_ptr = args->data_ptr;
 915         int do_bit17_swizzling;
 916
 917         remain = args->size;
 918
 919         /* Pin the user pages containing the data.  We can't fault while
 920          * holding the struct mutex, and all of the pwrite implementations
 921          * want to hold it while dereferencing the user data.
 922          */
 923         first_data_page = data_ptr / PAGE_SIZE;
 924         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 925         num_pages = last_data_page - first_data_page + 1;
 926
 927         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 928         if (user_pages == NULL)
 929                 return -ENOMEM;
 930
 931         mutex_unlock(&dev->struct_mutex);
 932         down_read(&mm->mmap_sem);
 933         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 934                                       num_pages, 0, 0, user_pages, NULL);
 935         up_read(&mm->mmap_sem);
 936         mutex_lock(&dev->struct_mutex);
 937         if (pinned_pages < num_pages) {
 938                 ret = -EFAULT;
 939                 goto out;
 940         }
 941
 942         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 943         if (ret)
 944                 goto out;
 945
 946         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 947
 948         obj_priv = to_intel_bo(obj);
 949         offset = args->offset;
 950         obj_priv->dirty = 1;
 951
 952         while (remain > 0) {
 953                 struct page *page;
 954
 955                 /* Operation in this page
 956                  *
 957                  * shmem_page_offset = offset within page in shmem file
 958                  * data_page_index = page number in get_user_pages return
 959                  * data_page_offset = offset with data_page_index page.
 960                  * page_length = bytes to copy for this page
 961                  */
 962                 shmem_page_offset = offset & ~PAGE_MASK;
 963                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 964                 data_page_offset = data_ptr & ~PAGE_MASK;
 965
 966                 page_length = remain;
 967                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 968                         page_length = PAGE_SIZE - shmem_page_offset;
 969                 if ((data_page_offset + page_length) > PAGE_SIZE)
 970                         page_length = PAGE_SIZE - data_page_offset;
 971
 972                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 973                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 974                 if (IS_ERR(page)) {
 975                         ret = PTR_ERR(page);
 976                         goto out;
 977                 }
 978
 979                 if (do_bit17_swizzling) {
 980                         slow_shmem_bit17_copy(page,
 981                                               shmem_page_offset,
 982                                               user_pages[data_page_index],
 983                                               data_page_offset,
 984                                               page_length,
 985                                               0);
 986                 } else {
 987                         slow_shmem_copy(page,
 988                                         shmem_page_offset,
 989                                         user_pages[data_page_index],
 990                                         data_page_offset,
 991                                         page_length);
 992                 }
 993
 994                 set_page_dirty(page);
 995                 mark_page_accessed(page);
 996                 page_cache_release(page);
 997
 998                 remain -= page_length;
 999                 data_ptr += page_length;
1000                 offset += page_length;
1001         }
1002
1003 out:
1004         for (i = 0; i < pinned_pages; i++)
1005                 page_cache_release(user_pages[i]);
1006         drm_free_large(user_pages);
1007
1008         return ret;
1009 }
1010
1011 /**
1012  * Writes data to the object referenced by handle.
1013  *
1014  * On error, the contents of the buffer that were to be modified are undefined.
1015  */
1016 int
1017 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1018                       struct drm_file *file)
1019 {
1020         struct drm_i915_gem_pwrite *args = data;
1021         struct drm_gem_object *obj;
1022         struct drm_i915_gem_object *obj_priv;
1023         int ret = 0;
1024
1025         ret = i915_mutex_lock_interruptible(dev);
1026         if (ret)
1027                 return ret;
1028
1029         obj = drm_gem_object_lookup(dev, file, args->handle);
1030         if (obj == NULL) {
1031                 ret = -ENOENT;
1032                 goto unlock;
1033         }
1034         obj_priv = to_intel_bo(obj);
1035
1036
1037         /* Bounds check destination. */
1038         if (args->offset > obj->size || args->size > obj->size - args->offset) {
1039                 ret = -EINVAL;
1040                 goto out;
1041         }
1042
1043         if (args->size == 0)
1044                 goto out;
1045
1046         if (!access_ok(VERIFY_READ,
1047                        (char __user *)(uintptr_t)args->data_ptr,
1048                        args->size)) {
1049                 ret = -EFAULT;
1050                 goto out;
1051         }
1052
1053         ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1054                                       args->size);
1055         if (ret) {
1056                 ret = -EFAULT;
1057                 goto out;
1058         }
1059
1060         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1061          * it would end up going through the fenced access, and we'll get
1062          * different detiling behavior between reading and writing.
1063          * pread/pwrite currently are reading and writing from the CPU
1064          * perspective, requiring manual detiling by the client.
1065          */
1066         if (obj_priv->phys_obj)
1067                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1068         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1069                  obj_priv->gtt_space &&
1070                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
1071                 ret = i915_gem_object_pin(obj, 0, true, false);
1072                 if (ret)
1073                         goto out;
1074
1075                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1076                 if (ret)
1077                         goto out_unpin;
1078
1079                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1080                 if (ret == -EFAULT)
1081                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1082
1083 out_unpin:
1084                 i915_gem_object_unpin(obj);
1085         } else {
1086                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1087                 if (ret)
1088                         goto out;
1089
1090                 ret = -EFAULT;
1091                 if (!i915_gem_object_needs_bit17_swizzle(obj))
1092                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1093                 if (ret == -EFAULT)
1094                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1095         }
1096
1097 out:
1098         drm_gem_object_unreference(obj);
1099 unlock:
1100         mutex_unlock(&dev->struct_mutex);
1101         return ret;
1102 }
1103
1104 /**
1105  * Called when user space prepares to use an object with the CPU, either
1106  * through the mmap ioctl's mapping or a GTT mapping.
1107  */
1108 int
1109 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1110                           struct drm_file *file_priv)
1111 {
1112         struct drm_i915_private *dev_priv = dev->dev_private;
1113         struct drm_i915_gem_set_domain *args = data;
1114         struct drm_gem_object *obj;
1115         struct drm_i915_gem_object *obj_priv;
1116         uint32_t read_domains = args->read_domains;
1117         uint32_t write_domain = args->write_domain;
1118         int ret;
1119
1120         if (!(dev->driver->driver_features & DRIVER_GEM))
1121                 return -ENODEV;
1122
1123         /* Only handle setting domains to types used by the CPU. */
1124         if (write_domain & I915_GEM_GPU_DOMAINS)
1125                 return -EINVAL;
1126
1127         if (read_domains & I915_GEM_GPU_DOMAINS)
1128                 return -EINVAL;
1129
1130         /* Having something in the write domain implies it's in the read
1131          * domain, and only that read domain.  Enforce that in the request.
1132          */
1133         if (write_domain != 0 && read_domains != write_domain)
1134                 return -EINVAL;
1135
1136         ret = i915_mutex_lock_interruptible(dev);
1137         if (ret)
1138                 return ret;
1139
1140         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1141         if (obj == NULL) {
1142                 ret = -ENOENT;
1143                 goto unlock;
1144         }
1145         obj_priv = to_intel_bo(obj);
1146
1147         intel_mark_busy(dev, obj);
1148
1149         if (read_domains & I915_GEM_DOMAIN_GTT) {
1150                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1151
1152                 /* Update the LRU on the fence for the CPU access that's
1153                  * about to occur.
1154                  */
1155                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1156                         struct drm_i915_fence_reg *reg =
1157                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1158                         list_move_tail(&reg->lru_list,
1159                                        &dev_priv->mm.fence_list);
1160                 }
1161
1162                 /* Silently promote "you're not bound, there was nothing to do"
1163                  * to success, since the client was just asking us to
1164                  * make sure everything was done.
1165                  */
1166                 if (ret == -EINVAL)
1167                         ret = 0;
1168         } else {
1169                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1170         }
1171
1172         /* Maintain LRU order of "inactive" objects */
1173         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1174                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1175
1176         drm_gem_object_unreference(obj);
1177 unlock:
1178         mutex_unlock(&dev->struct_mutex);
1179         return ret;
1180 }
1181
1182 /**
1183  * Called when user space has done writes to this buffer
1184  */
1185 int
1186 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1187                       struct drm_file *file_priv)
1188 {
1189         struct drm_i915_gem_sw_finish *args = data;
1190         struct drm_gem_object *obj;
1191         int ret = 0;
1192
1193         if (!(dev->driver->driver_features & DRIVER_GEM))
1194                 return -ENODEV;
1195
1196         ret = i915_mutex_lock_interruptible(dev);
1197         if (ret)
1198                 return ret;
1199
1200         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1201         if (obj == NULL) {
1202                 ret = -ENOENT;
1203                 goto unlock;
1204         }
1205
1206         /* Pinned buffers may be scanout, so flush the cache */
1207         if (to_intel_bo(obj)->pin_count)
1208                 i915_gem_object_flush_cpu_write_domain(obj);
1209
1210         drm_gem_object_unreference(obj);
1211 unlock:
1212         mutex_unlock(&dev->struct_mutex);
1213         return ret;
1214 }
1215
1216 /**
1217  * Maps the contents of an object, returning the address it is mapped
1218  * into.
1219  *
1220  * While the mapping holds a reference on the contents of the object, it doesn't
1221  * imply a ref on the object itself.
1222  */
1223 int
1224 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1225                    struct drm_file *file_priv)
1226 {
1227         struct drm_i915_private *dev_priv = dev->dev_private;
1228         struct drm_i915_gem_mmap *args = data;
1229         struct drm_gem_object *obj;
1230         loff_t offset;
1231         unsigned long addr;
1232
1233         if (!(dev->driver->driver_features & DRIVER_GEM))
1234                 return -ENODEV;
1235
1236         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1237         if (obj == NULL)
1238                 return -ENOENT;
1239
1240         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1241                 drm_gem_object_unreference_unlocked(obj);
1242                 return -E2BIG;
1243         }
1244
1245         offset = args->offset;
1246
1247         down_write(&current->mm->mmap_sem);
1248         addr = do_mmap(obj->filp, 0, args->size,
1249                        PROT_READ | PROT_WRITE, MAP_SHARED,
1250                        args->offset);
1251         up_write(&current->mm->mmap_sem);
1252         drm_gem_object_unreference_unlocked(obj);
1253         if (IS_ERR((void *)addr))
1254                 return addr;
1255
1256         args->addr_ptr = (uint64_t) addr;
1257
1258         return 0;
1259 }
1260
1261 /**
1262  * i915_gem_fault - fault a page into the GTT
1263  * vma: VMA in question
1264  * vmf: fault info
1265  *
1266  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1267  * from userspace.  The fault handler takes care of binding the object to
1268  * the GTT (if needed), allocating and programming a fence register (again,
1269  * only if needed based on whether the old reg is still valid or the object
1270  * is tiled) and inserting a new PTE into the faulting process.
1271  *
1272  * Note that the faulting process may involve evicting existing objects
1273  * from the GTT and/or fence registers to make room.  So performance may
1274  * suffer if the GTT working set is large or there are few fence registers
1275  * left.
1276  */
1277 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1278 {
1279         struct drm_gem_object *obj = vma->vm_private_data;
1280         struct drm_device *dev = obj->dev;
1281         drm_i915_private_t *dev_priv = dev->dev_private;
1282         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1283         pgoff_t page_offset;
1284         unsigned long pfn;
1285         int ret = 0;
1286         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1287
1288         /* We don't use vmf->pgoff since that has the fake offset */
1289         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1290                 PAGE_SHIFT;
1291
1292         /* Now bind it into the GTT if needed */
1293         mutex_lock(&dev->struct_mutex);
1294         BUG_ON(obj_priv->pin_count && !obj_priv->pin_mappable);
1295
1296         if (obj_priv->gtt_space) {
1297                 if (!obj_priv->mappable ||
1298                     (obj_priv->tiling_mode && !obj_priv->fenceable)) {
1299                         ret = i915_gem_object_unbind(obj);
1300                         if (ret)
1301                                 goto unlock;
1302                 }
1303         }
1304
1305         if (!obj_priv->gtt_space) {
1306                 ret = i915_gem_object_bind_to_gtt(obj, 0,
1307                                                   true, obj_priv->tiling_mode);
1308                 if (ret)
1309                         goto unlock;
1310         }
1311
1312         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1313         if (ret)
1314                 goto unlock;
1315
1316         if (!obj_priv->fault_mappable) {
1317                 obj_priv->fault_mappable = true;
1318                 i915_gem_info_update_mappable(dev_priv, obj_priv, true);
1319         }
1320
1321         /* Need a new fence register? */
1322         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1323                 ret = i915_gem_object_get_fence_reg(obj, true);
1324                 if (ret)
1325                         goto unlock;
1326         }
1327
1328         if (i915_gem_object_is_inactive(obj_priv))
1329                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1330
1331         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1332                 page_offset;
1333
1334         /* Finally, remap it using the new GTT offset */
1335         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1336 unlock:
1337         mutex_unlock(&dev->struct_mutex);
1338
1339         switch (ret) {
1340         case 0:
1341         case -ERESTARTSYS:
1342                 return VM_FAULT_NOPAGE;
1343         case -ENOMEM:
1344         case -EAGAIN:
1345                 return VM_FAULT_OOM;
1346         default:
1347                 return VM_FAULT_SIGBUS;
1348         }
1349 }
1350
1351 /**
1352  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1353  * @obj: obj in question
1354  *
1355  * GEM memory mapping works by handing back to userspace a fake mmap offset
1356  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1357  * up the object based on the offset and sets up the various memory mapping
1358  * structures.
1359  *
1360  * This routine allocates and attaches a fake offset for @obj.
1361  */
1362 static int
1363 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1364 {
1365         struct drm_device *dev = obj->dev;
1366         struct drm_gem_mm *mm = dev->mm_private;
1367         struct drm_map_list *list;
1368         struct drm_local_map *map;
1369         int ret = 0;
1370
1371         /* Set the object up for mmap'ing */
1372         list = &obj->map_list;
1373         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1374         if (!list->map)
1375                 return -ENOMEM;
1376
1377         map = list->map;
1378         map->type = _DRM_GEM;
1379         map->size = obj->size;
1380         map->handle = obj;
1381
1382         /* Get a DRM GEM mmap offset allocated... */
1383         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1384                                                     obj->size / PAGE_SIZE, 0, 0);
1385         if (!list->file_offset_node) {
1386                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1387                 ret = -ENOSPC;
1388                 goto out_free_list;
1389         }
1390
1391         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1392                                                   obj->size / PAGE_SIZE, 0);
1393         if (!list->file_offset_node) {
1394                 ret = -ENOMEM;
1395                 goto out_free_list;
1396         }
1397
1398         list->hash.key = list->file_offset_node->start;
1399         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1400         if (ret) {
1401                 DRM_ERROR("failed to add to map hash\n");
1402                 goto out_free_mm;
1403         }
1404
1405         return 0;
1406
1407 out_free_mm:
1408         drm_mm_put_block(list->file_offset_node);
1409 out_free_list:
1410         kfree(list->map);
1411         list->map = NULL;
1412
1413         return ret;
1414 }
1415
1416 /**
1417  * i915_gem_release_mmap - remove physical page mappings
1418  * @obj: obj in question
1419  *
1420  * Preserve the reservation of the mmapping with the DRM core code, but
1421  * relinquish ownership of the pages back to the system.
1422  *
1423  * It is vital that we remove the page mapping if we have mapped a tiled
1424  * object through the GTT and then lose the fence register due to
1425  * resource pressure. Similarly if the object has been moved out of the
1426  * aperture, than pages mapped into userspace must be revoked. Removing the
1427  * mapping will then trigger a page fault on the next user access, allowing
1428  * fixup by i915_gem_fault().
1429  */
1430 void
1431 i915_gem_release_mmap(struct drm_gem_object *obj)
1432 {
1433         struct drm_device *dev = obj->dev;
1434         struct drm_i915_private *dev_priv = dev->dev_private;
1435         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1436
1437         if (unlikely(obj->map_list.map && dev->dev_mapping))
1438                 unmap_mapping_range(dev->dev_mapping,
1439                                     (loff_t)obj->map_list.hash.key<<PAGE_SHIFT,
1440                                     obj->size, 1);
1441
1442         if (obj_priv->fault_mappable) {
1443                 obj_priv->fault_mappable = false;
1444                 i915_gem_info_update_mappable(dev_priv, obj_priv, false);
1445         }
1446 }
1447
1448 static void
1449 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1450 {
1451         struct drm_device *dev = obj->dev;
1452         struct drm_gem_mm *mm = dev->mm_private;
1453         struct drm_map_list *list = &obj->map_list;
1454
1455         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1456         drm_mm_put_block(list->file_offset_node);
1457         kfree(list->map);
1458         list->map = NULL;
1459 }
1460
1461 /**
1462  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1463  * @obj: object to check
1464  *
1465  * Return the required GTT alignment for an object, taking into account
1466  * potential fence register mapping if needed.
1467  */
1468 static uint32_t
1469 i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv)
1470 {
1471         struct drm_device *dev = obj_priv->base.dev;
1472
1473         /*
1474          * Minimum alignment is 4k (GTT page size), but might be greater
1475          * if a fence register is needed for the object.
1476          */
1477         if (INTEL_INFO(dev)->gen >= 4 ||
1478             obj_priv->tiling_mode == I915_TILING_NONE)
1479                 return 4096;
1480
1481         /*
1482          * Previous chips need to be aligned to the size of the smallest
1483          * fence register that can contain the object.
1484          */
1485         return i915_gem_get_gtt_size(obj_priv);
1486 }
1487
1488 static uint32_t
1489 i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv)
1490 {
1491         struct drm_device *dev = obj_priv->base.dev;
1492         uint32_t size;
1493
1494         /*
1495          * Minimum alignment is 4k (GTT page size), but might be greater
1496          * if a fence register is needed for the object.
1497          */
1498         if (INTEL_INFO(dev)->gen >= 4)
1499                 return obj_priv->base.size;
1500
1501         /*
1502          * Previous chips need to be aligned to the size of the smallest
1503          * fence register that can contain the object.
1504          */
1505         if (INTEL_INFO(dev)->gen == 3)
1506                 size = 1024*1024;
1507         else
1508                 size = 512*1024;
1509
1510         while (size < obj_priv->base.size)
1511                 size <<= 1;
1512
1513         return size;
1514 }
1515
1516 /**
1517  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1518  * @dev: DRM device
1519  * @data: GTT mapping ioctl data
1520  * @file_priv: GEM object info
1521  *
1522  * Simply returns the fake offset to userspace so it can mmap it.
1523  * The mmap call will end up in drm_gem_mmap(), which will set things
1524  * up so we can get faults in the handler above.
1525  *
1526  * The fault handler will take care of binding the object into the GTT
1527  * (since it may have been evicted to make room for something), allocating
1528  * a fence register, and mapping the appropriate aperture address into
1529  * userspace.
1530  */
1531 int
1532 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1533                         struct drm_file *file_priv)
1534 {
1535         struct drm_i915_private *dev_priv = dev->dev_private;
1536         struct drm_i915_gem_mmap_gtt *args = data;
1537         struct drm_gem_object *obj;
1538         struct drm_i915_gem_object *obj_priv;
1539         int ret;
1540
1541         if (!(dev->driver->driver_features & DRIVER_GEM))
1542                 return -ENODEV;
1543
1544         ret = i915_mutex_lock_interruptible(dev);
1545         if (ret)
1546                 return ret;
1547
1548         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1549         if (obj == NULL) {
1550                 ret = -ENOENT;
1551                 goto unlock;
1552         }
1553         obj_priv = to_intel_bo(obj);
1554
1555         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1556                 ret = -E2BIG;
1557                 goto unlock;
1558         }
1559
1560         if (obj_priv->madv != I915_MADV_WILLNEED) {
1561                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1562                 ret = -EINVAL;
1563                 goto out;
1564         }
1565
1566         if (!obj->map_list.map) {
1567                 ret = i915_gem_create_mmap_offset(obj);
1568                 if (ret)
1569                         goto out;
1570         }
1571
1572         args->offset = (u64)obj->map_list.hash.key << PAGE_SHIFT;
1573
1574 out:
1575         drm_gem_object_unreference(obj);
1576 unlock:
1577         mutex_unlock(&dev->struct_mutex);
1578         return ret;
1579 }
1580
1581 static int
1582 i915_gem_object_get_pages_gtt(struct drm_gem_object *obj,
1583                               gfp_t gfpmask)
1584 {
1585         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1586         int page_count, i;
1587         struct address_space *mapping;
1588         struct inode *inode;
1589         struct page *page;
1590
1591         /* Get the list of pages out of our struct file.  They'll be pinned
1592          * at this point until we release them.
1593          */
1594         page_count = obj->size / PAGE_SIZE;
1595         BUG_ON(obj_priv->pages != NULL);
1596         obj_priv->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1597         if (obj_priv->pages == NULL)
1598                 return -ENOMEM;
1599
1600         inode = obj->filp->f_path.dentry->d_inode;
1601         mapping = inode->i_mapping;
1602         for (i = 0; i < page_count; i++) {
1603                 page = read_cache_page_gfp(mapping, i,
1604                                            GFP_HIGHUSER |
1605                                            __GFP_COLD |
1606                                            __GFP_RECLAIMABLE |
1607                                            gfpmask);
1608                 if (IS_ERR(page))
1609                         goto err_pages;
1610
1611                 obj_priv->pages[i] = page;
1612         }
1613
1614         if (obj_priv->tiling_mode != I915_TILING_NONE)
1615                 i915_gem_object_do_bit_17_swizzle(obj);
1616
1617         return 0;
1618
1619 err_pages:
1620         while (i--)
1621                 page_cache_release(obj_priv->pages[i]);
1622
1623         drm_free_large(obj_priv->pages);
1624         obj_priv->pages = NULL;
1625         return PTR_ERR(page);
1626 }
1627
1628 static void
1629 i915_gem_object_put_pages_gtt(struct drm_gem_object *obj)
1630 {
1631         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1632         int page_count = obj->size / PAGE_SIZE;
1633         int i;
1634
1635         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1636
1637         if (obj_priv->tiling_mode != I915_TILING_NONE)
1638                 i915_gem_object_save_bit_17_swizzle(obj);
1639
1640         if (obj_priv->madv == I915_MADV_DONTNEED)
1641                 obj_priv->dirty = 0;
1642
1643         for (i = 0; i < page_count; i++) {
1644                 if (obj_priv->dirty)
1645                         set_page_dirty(obj_priv->pages[i]);
1646
1647                 if (obj_priv->madv == I915_MADV_WILLNEED)
1648                         mark_page_accessed(obj_priv->pages[i]);
1649
1650                 page_cache_release(obj_priv->pages[i]);
1651         }
1652         obj_priv->dirty = 0;
1653
1654         drm_free_large(obj_priv->pages);
1655         obj_priv->pages = NULL;
1656 }
1657
1658 static uint32_t
1659 i915_gem_next_request_seqno(struct drm_device *dev,
1660                             struct intel_ring_buffer *ring)
1661 {
1662         drm_i915_private_t *dev_priv = dev->dev_private;
1663
1664         ring->outstanding_lazy_request = true;
1665         return dev_priv->next_seqno;
1666 }
1667
1668 static void
1669 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1670                                struct intel_ring_buffer *ring)
1671 {
1672         struct drm_device *dev = obj->dev;
1673         struct drm_i915_private *dev_priv = dev->dev_private;
1674         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1675         uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1676
1677         BUG_ON(ring == NULL);
1678         obj_priv->ring = ring;
1679
1680         /* Add a reference if we're newly entering the active list. */
1681         if (!obj_priv->active) {
1682                 drm_gem_object_reference(obj);
1683                 obj_priv->active = 1;
1684         }
1685
1686         /* Move from whatever list we were on to the tail of execution. */
1687         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1688         list_move_tail(&obj_priv->ring_list, &ring->active_list);
1689         obj_priv->last_rendering_seqno = seqno;
1690 }
1691
1692 static void
1693 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1694 {
1695         struct drm_device *dev = obj->dev;
1696         drm_i915_private_t *dev_priv = dev->dev_private;
1697         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1698
1699         BUG_ON(!obj_priv->active);
1700         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1701         list_del_init(&obj_priv->ring_list);
1702         obj_priv->last_rendering_seqno = 0;
1703 }
1704
1705 /* Immediately discard the backing storage */
1706 static void
1707 i915_gem_object_truncate(struct drm_gem_object *obj)
1708 {
1709         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1710         struct inode *inode;
1711
1712         /* Our goal here is to return as much of the memory as
1713          * is possible back to the system as we are called from OOM.
1714          * To do this we must instruct the shmfs to drop all of its
1715          * backing pages, *now*. Here we mirror the actions taken
1716          * when by shmem_delete_inode() to release the backing store.
1717          */
1718         inode = obj->filp->f_path.dentry->d_inode;
1719         truncate_inode_pages(inode->i_mapping, 0);
1720         if (inode->i_op->truncate_range)
1721                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1722
1723         obj_priv->madv = __I915_MADV_PURGED;
1724 }
1725
1726 static inline int
1727 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1728 {
1729         return obj_priv->madv == I915_MADV_DONTNEED;
1730 }
1731
1732 static void
1733 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1734 {
1735         struct drm_device *dev = obj->dev;
1736         drm_i915_private_t *dev_priv = dev->dev_private;
1737         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1738
1739         if (obj_priv->pin_count != 0)
1740                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1741         else
1742                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1743         list_del_init(&obj_priv->ring_list);
1744
1745         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1746
1747         obj_priv->last_rendering_seqno = 0;
1748         obj_priv->ring = NULL;
1749         if (obj_priv->active) {
1750                 obj_priv->active = 0;
1751                 drm_gem_object_unreference(obj);
1752         }
1753         WARN_ON(i915_verify_lists(dev));
1754 }
1755
1756 static void
1757 i915_gem_process_flushing_list(struct drm_device *dev,
1758                                uint32_t flush_domains,
1759                                struct intel_ring_buffer *ring)
1760 {
1761         drm_i915_private_t *dev_priv = dev->dev_private;
1762         struct drm_i915_gem_object *obj_priv, *next;
1763
1764         list_for_each_entry_safe(obj_priv, next,
1765                                  &ring->gpu_write_list,
1766                                  gpu_write_list) {
1767                 struct drm_gem_object *obj = &obj_priv->base;
1768
1769                 if (obj->write_domain & flush_domains) {
1770                         uint32_t old_write_domain = obj->write_domain;
1771
1772                         obj->write_domain = 0;
1773                         list_del_init(&obj_priv->gpu_write_list);
1774                         i915_gem_object_move_to_active(obj, ring);
1775
1776                         /* update the fence lru list */
1777                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1778                                 struct drm_i915_fence_reg *reg =
1779                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1780                                 list_move_tail(&reg->lru_list,
1781                                                 &dev_priv->mm.fence_list);
1782                         }
1783
1784                         trace_i915_gem_object_change_domain(obj,
1785                                                             obj->read_domains,
1786                                                             old_write_domain);
1787                 }
1788         }
1789 }
1790
1791 int
1792 i915_add_request(struct drm_device *dev,
1793                  struct drm_file *file,
1794                  struct drm_i915_gem_request *request,
1795                  struct intel_ring_buffer *ring)
1796 {
1797         drm_i915_private_t *dev_priv = dev->dev_private;
1798         struct drm_i915_file_private *file_priv = NULL;
1799         uint32_t seqno;
1800         int was_empty;
1801         int ret;
1802
1803         BUG_ON(request == NULL);
1804
1805         if (file != NULL)
1806                 file_priv = file->driver_priv;
1807
1808         ret = ring->add_request(ring, &seqno);
1809         if (ret)
1810             return ret;
1811
1812         ring->outstanding_lazy_request = false;
1813
1814         request->seqno = seqno;
1815         request->ring = ring;
1816         request->emitted_jiffies = jiffies;
1817         was_empty = list_empty(&ring->request_list);
1818         list_add_tail(&request->list, &ring->request_list);
1819
1820         if (file_priv) {
1821                 spin_lock(&file_priv->mm.lock);
1822                 request->file_priv = file_priv;
1823                 list_add_tail(&request->client_list,
1824                               &file_priv->mm.request_list);
1825                 spin_unlock(&file_priv->mm.lock);
1826         }
1827
1828         if (!dev_priv->mm.suspended) {
1829                 mod_timer(&dev_priv->hangcheck_timer,
1830                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1831                 if (was_empty)
1832                         queue_delayed_work(dev_priv->wq,
1833                                            &dev_priv->mm.retire_work, HZ);
1834         }
1835         return 0;
1836 }
1837
1838 /**
1839  * Command execution barrier
1840  *
1841  * Ensures that all commands in the ring are finished
1842  * before signalling the CPU
1843  */
1844 static void
1845 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1846 {
1847         uint32_t flush_domains = 0;
1848
1849         /* The sampler always gets flushed on i965 (sigh) */
1850         if (INTEL_INFO(dev)->gen >= 4)
1851                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1852
1853         ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
1854 }
1855
1856 static inline void
1857 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1858 {
1859         struct drm_i915_file_private *file_priv = request->file_priv;
1860
1861         if (!file_priv)
1862                 return;
1863
1864         spin_lock(&file_priv->mm.lock);
1865         list_del(&request->client_list);
1866         request->file_priv = NULL;
1867         spin_unlock(&file_priv->mm.lock);
1868 }
1869
1870 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1871                                       struct intel_ring_buffer *ring)
1872 {
1873         while (!list_empty(&ring->request_list)) {
1874                 struct drm_i915_gem_request *request;
1875
1876                 request = list_first_entry(&ring->request_list,
1877                                            struct drm_i915_gem_request,
1878                                            list);
1879
1880                 list_del(&request->list);
1881                 i915_gem_request_remove_from_client(request);
1882                 kfree(request);
1883         }
1884
1885         while (!list_empty(&ring->active_list)) {
1886                 struct drm_i915_gem_object *obj_priv;
1887
1888                 obj_priv = list_first_entry(&ring->active_list,
1889                                             struct drm_i915_gem_object,
1890                                             ring_list);
1891
1892                 obj_priv->base.write_domain = 0;
1893                 list_del_init(&obj_priv->gpu_write_list);
1894                 i915_gem_object_move_to_inactive(&obj_priv->base);
1895         }
1896 }
1897
1898 void i915_gem_reset(struct drm_device *dev)
1899 {
1900         struct drm_i915_private *dev_priv = dev->dev_private;
1901         struct drm_i915_gem_object *obj_priv;
1902         int i;
1903
1904         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1905         i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1906         i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1907
1908         /* Remove anything from the flushing lists. The GPU cache is likely
1909          * to be lost on reset along with the data, so simply move the
1910          * lost bo to the inactive list.
1911          */
1912         while (!list_empty(&dev_priv->mm.flushing_list)) {
1913                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1914                                             struct drm_i915_gem_object,
1915                                             mm_list);
1916
1917                 obj_priv->base.write_domain = 0;
1918                 list_del_init(&obj_priv->gpu_write_list);
1919                 i915_gem_object_move_to_inactive(&obj_priv->base);
1920         }
1921
1922         /* Move everything out of the GPU domains to ensure we do any
1923          * necessary invalidation upon reuse.
1924          */
1925         list_for_each_entry(obj_priv,
1926                             &dev_priv->mm.inactive_list,
1927                             mm_list)
1928         {
1929                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1930         }
1931
1932         /* The fence registers are invalidated so clear them out */
1933         for (i = 0; i < 16; i++) {
1934                 struct drm_i915_fence_reg *reg;
1935
1936                 reg = &dev_priv->fence_regs[i];
1937                 if (!reg->obj)
1938                         continue;
1939
1940                 i915_gem_clear_fence_reg(reg->obj);
1941         }
1942 }
1943
1944 /**
1945  * This function clears the request list as sequence numbers are passed.
1946  */
1947 static void
1948 i915_gem_retire_requests_ring(struct drm_device *dev,
1949                               struct intel_ring_buffer *ring)
1950 {
1951         drm_i915_private_t *dev_priv = dev->dev_private;
1952         uint32_t seqno;
1953
1954         if (!ring->status_page.page_addr ||
1955             list_empty(&ring->request_list))
1956                 return;
1957
1958         WARN_ON(i915_verify_lists(dev));
1959
1960         seqno = ring->get_seqno(ring);
1961         while (!list_empty(&ring->request_list)) {
1962                 struct drm_i915_gem_request *request;
1963
1964                 request = list_first_entry(&ring->request_list,
1965                                            struct drm_i915_gem_request,
1966                                            list);
1967
1968                 if (!i915_seqno_passed(seqno, request->seqno))
1969                         break;
1970
1971                 trace_i915_gem_request_retire(dev, request->seqno);
1972
1973                 list_del(&request->list);
1974                 i915_gem_request_remove_from_client(request);
1975                 kfree(request);
1976         }
1977
1978         /* Move any buffers on the active list that are no longer referenced
1979          * by the ringbuffer to the flushing/inactive lists as appropriate.
1980          */
1981         while (!list_empty(&ring->active_list)) {
1982                 struct drm_gem_object *obj;
1983                 struct drm_i915_gem_object *obj_priv;
1984
1985                 obj_priv = list_first_entry(&ring->active_list,
1986                                             struct drm_i915_gem_object,
1987                                             ring_list);
1988
1989                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1990                         break;
1991
1992                 obj = &obj_priv->base;
1993                 if (obj->write_domain != 0)
1994                         i915_gem_object_move_to_flushing(obj);
1995                 else
1996                         i915_gem_object_move_to_inactive(obj);
1997         }
1998
1999         if (unlikely (dev_priv->trace_irq_seqno &&
2000                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
2001                 ring->user_irq_put(ring);
2002                 dev_priv->trace_irq_seqno = 0;
2003         }
2004
2005         WARN_ON(i915_verify_lists(dev));
2006 }
2007
2008 void
2009 i915_gem_retire_requests(struct drm_device *dev)
2010 {
2011         drm_i915_private_t *dev_priv = dev->dev_private;
2012
2013         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2014             struct drm_i915_gem_object *obj_priv, *tmp;
2015
2016             /* We must be careful that during unbind() we do not
2017              * accidentally infinitely recurse into retire requests.
2018              * Currently:
2019              *   retire -> free -> unbind -> wait -> retire_ring
2020              */
2021             list_for_each_entry_safe(obj_priv, tmp,
2022                                      &dev_priv->mm.deferred_free_list,
2023                                      mm_list)
2024                     i915_gem_free_object_tail(&obj_priv->base);
2025         }
2026
2027         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
2028         i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
2029         i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
2030 }
2031
2032 static void
2033 i915_gem_retire_work_handler(struct work_struct *work)
2034 {
2035         drm_i915_private_t *dev_priv;
2036         struct drm_device *dev;
2037
2038         dev_priv = container_of(work, drm_i915_private_t,
2039                                 mm.retire_work.work);
2040         dev = dev_priv->dev;
2041
2042         /* Come back later if the device is busy... */
2043         if (!mutex_trylock(&dev->struct_mutex)) {
2044                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2045                 return;
2046         }
2047
2048         i915_gem_retire_requests(dev);
2049
2050         if (!dev_priv->mm.suspended &&
2051                 (!list_empty(&dev_priv->render_ring.request_list) ||
2052                  !list_empty(&dev_priv->bsd_ring.request_list) ||
2053                  !list_empty(&dev_priv->blt_ring.request_list)))
2054                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2055         mutex_unlock(&dev->struct_mutex);
2056 }
2057
2058 int
2059 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
2060                      bool interruptible, struct intel_ring_buffer *ring)
2061 {
2062         drm_i915_private_t *dev_priv = dev->dev_private;
2063         u32 ier;
2064         int ret = 0;
2065
2066         BUG_ON(seqno == 0);
2067
2068         if (atomic_read(&dev_priv->mm.wedged))
2069                 return -EAGAIN;
2070
2071         if (ring->outstanding_lazy_request) {
2072                 struct drm_i915_gem_request *request;
2073
2074                 request = kzalloc(sizeof(*request), GFP_KERNEL);
2075                 if (request == NULL)
2076                         return -ENOMEM;
2077
2078                 ret = i915_add_request(dev, NULL, request, ring);
2079                 if (ret) {
2080                         kfree(request);
2081                         return ret;
2082                 }
2083
2084                 seqno = request->seqno;
2085         }
2086         BUG_ON(seqno == dev_priv->next_seqno);
2087
2088         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2089                 if (HAS_PCH_SPLIT(dev))
2090                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2091                 else
2092                         ier = I915_READ(IER);
2093                 if (!ier) {
2094                         DRM_ERROR("something (likely vbetool) disabled "
2095                                   "interrupts, re-enabling\n");
2096                         i915_driver_irq_preinstall(dev);
2097                         i915_driver_irq_postinstall(dev);
2098                 }
2099
2100                 trace_i915_gem_request_wait_begin(dev, seqno);
2101
2102                 ring->waiting_seqno = seqno;
2103                 ring->user_irq_get(ring);
2104                 if (interruptible)
2105                         ret = wait_event_interruptible(ring->irq_queue,
2106                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2107                                 || atomic_read(&dev_priv->mm.wedged));
2108                 else
2109                         wait_event(ring->irq_queue,
2110                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2111                                 || atomic_read(&dev_priv->mm.wedged));
2112
2113                 ring->user_irq_put(ring);
2114                 ring->waiting_seqno = 0;
2115
2116                 trace_i915_gem_request_wait_end(dev, seqno);
2117         }
2118         if (atomic_read(&dev_priv->mm.wedged))
2119                 ret = -EAGAIN;
2120
2121         if (ret && ret != -ERESTARTSYS)
2122                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2123                           __func__, ret, seqno, ring->get_seqno(ring),
2124                           dev_priv->next_seqno);
2125
2126         /* Directly dispatch request retiring.  While we have the work queue
2127          * to handle this, the waiter on a request often wants an associated
2128          * buffer to have made it to the inactive list, and we would need
2129          * a separate wait queue to handle that.
2130          */
2131         if (ret == 0)
2132                 i915_gem_retire_requests_ring(dev, ring);
2133
2134         return ret;
2135 }
2136
2137 /**
2138  * Waits for a sequence number to be signaled, and cleans up the
2139  * request and object lists appropriately for that event.
2140  */
2141 static int
2142 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2143                   struct intel_ring_buffer *ring)
2144 {
2145         return i915_do_wait_request(dev, seqno, 1, ring);
2146 }
2147
2148 static void
2149 i915_gem_flush_ring(struct drm_device *dev,
2150                     struct drm_file *file_priv,
2151                     struct intel_ring_buffer *ring,
2152                     uint32_t invalidate_domains,
2153                     uint32_t flush_domains)
2154 {
2155         ring->flush(ring, invalidate_domains, flush_domains);
2156         i915_gem_process_flushing_list(dev, flush_domains, ring);
2157 }
2158
2159 static void
2160 i915_gem_flush(struct drm_device *dev,
2161                struct drm_file *file_priv,
2162                uint32_t invalidate_domains,
2163                uint32_t flush_domains,
2164                uint32_t flush_rings)
2165 {
2166         drm_i915_private_t *dev_priv = dev->dev_private;
2167
2168         if (flush_domains & I915_GEM_DOMAIN_CPU)
2169                 drm_agp_chipset_flush(dev);
2170
2171         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2172                 if (flush_rings & RING_RENDER)
2173                         i915_gem_flush_ring(dev, file_priv,
2174                                             &dev_priv->render_ring,
2175                                             invalidate_domains, flush_domains);
2176                 if (flush_rings & RING_BSD)
2177                         i915_gem_flush_ring(dev, file_priv,
2178                                             &dev_priv->bsd_ring,
2179                                             invalidate_domains, flush_domains);
2180                 if (flush_rings & RING_BLT)
2181                         i915_gem_flush_ring(dev, file_priv,
2182                                             &dev_priv->blt_ring,
2183                                             invalidate_domains, flush_domains);
2184         }
2185 }
2186
2187 /**
2188  * Ensures that all rendering to the object has completed and the object is
2189  * safe to unbind from the GTT or access from the CPU.
2190  */
2191 static int
2192 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2193                                bool interruptible)
2194 {
2195         struct drm_device *dev = obj->dev;
2196         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2197         int ret;
2198
2199         /* This function only exists to support waiting for existing rendering,
2200          * not for emitting required flushes.
2201          */
2202         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2203
2204         /* If there is rendering queued on the buffer being evicted, wait for
2205          * it.
2206          */
2207         if (obj_priv->active) {
2208                 ret = i915_do_wait_request(dev,
2209                                            obj_priv->last_rendering_seqno,
2210                                            interruptible,
2211                                            obj_priv->ring);
2212                 if (ret)
2213                         return ret;
2214         }
2215
2216         return 0;
2217 }
2218
2219 /**
2220  * Unbinds an object from the GTT aperture.
2221  */
2222 int
2223 i915_gem_object_unbind(struct drm_gem_object *obj)
2224 {
2225         struct drm_device *dev = obj->dev;
2226         struct drm_i915_private *dev_priv = dev->dev_private;
2227         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2228         int ret = 0;
2229
2230         if (obj_priv->gtt_space == NULL)
2231                 return 0;
2232
2233         if (obj_priv->pin_count != 0) {
2234                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2235                 return -EINVAL;
2236         }
2237
2238         /* blow away mappings if mapped through GTT */
2239         i915_gem_release_mmap(obj);
2240
2241         /* Move the object to the CPU domain to ensure that
2242          * any possible CPU writes while it's not in the GTT
2243          * are flushed when we go to remap it. This will
2244          * also ensure that all pending GPU writes are finished
2245          * before we unbind.
2246          */
2247         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2248         if (ret == -ERESTARTSYS)
2249                 return ret;
2250         /* Continue on if we fail due to EIO, the GPU is hung so we
2251          * should be safe and we need to cleanup or else we might
2252          * cause memory corruption through use-after-free.
2253          */
2254         if (ret) {
2255                 i915_gem_clflush_object(obj);
2256                 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2257         }
2258
2259         /* release the fence reg _after_ flushing */
2260         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2261                 i915_gem_clear_fence_reg(obj);
2262
2263         drm_unbind_agp(obj_priv->agp_mem);
2264         drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2265
2266         i915_gem_object_put_pages_gtt(obj);
2267
2268         i915_gem_info_remove_gtt(dev_priv, obj_priv);
2269         list_del_init(&obj_priv->mm_list);
2270         obj_priv->fenceable = true;
2271         obj_priv->mappable = true;
2272
2273         drm_mm_put_block(obj_priv->gtt_space);
2274         obj_priv->gtt_space = NULL;
2275         obj_priv->gtt_offset = 0;
2276
2277         if (i915_gem_object_is_purgeable(obj_priv))
2278                 i915_gem_object_truncate(obj);
2279
2280         trace_i915_gem_object_unbind(obj);
2281
2282         return ret;
2283 }
2284
2285 static int i915_ring_idle(struct drm_device *dev,
2286                           struct intel_ring_buffer *ring)
2287 {
2288         if (list_empty(&ring->gpu_write_list))
2289                 return 0;
2290
2291         i915_gem_flush_ring(dev, NULL, ring,
2292                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2293         return i915_wait_request(dev,
2294                                  i915_gem_next_request_seqno(dev, ring),
2295                                  ring);
2296 }
2297
2298 int
2299 i915_gpu_idle(struct drm_device *dev)
2300 {
2301         drm_i915_private_t *dev_priv = dev->dev_private;
2302         bool lists_empty;
2303         int ret;
2304
2305         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2306                        list_empty(&dev_priv->render_ring.active_list) &&
2307                        list_empty(&dev_priv->bsd_ring.active_list) &&
2308                        list_empty(&dev_priv->blt_ring.active_list));
2309         if (lists_empty)
2310                 return 0;
2311
2312         /* Flush everything onto the inactive list. */
2313         ret = i915_ring_idle(dev, &dev_priv->render_ring);
2314         if (ret)
2315                 return ret;
2316
2317         ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2318         if (ret)
2319                 return ret;
2320
2321         ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2322         if (ret)
2323                 return ret;
2324
2325         return 0;
2326 }
2327
2328 static void sandybridge_write_fence_reg(struct drm_gem_object *obj)
2329 {
2330         struct drm_device *dev = obj->dev;
2331         drm_i915_private_t *dev_priv = dev->dev_private;
2332         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2333         u32 size = i915_gem_get_gtt_size(obj_priv);
2334         int regnum = obj_priv->fence_reg;
2335         uint64_t val;
2336
2337         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2338                     0xfffff000) << 32;
2339         val |= obj_priv->gtt_offset & 0xfffff000;
2340         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2341                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2342
2343         if (obj_priv->tiling_mode == I915_TILING_Y)
2344                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2345         val |= I965_FENCE_REG_VALID;
2346
2347         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2348 }
2349
2350 static void i965_write_fence_reg(struct drm_gem_object *obj)
2351 {
2352         struct drm_device *dev = obj->dev;
2353         drm_i915_private_t *dev_priv = dev->dev_private;
2354         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2355         u32 size = i915_gem_get_gtt_size(obj_priv);
2356         int regnum = obj_priv->fence_reg;
2357         uint64_t val;
2358
2359         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2360                     0xfffff000) << 32;
2361         val |= obj_priv->gtt_offset & 0xfffff000;
2362         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2363         if (obj_priv->tiling_mode == I915_TILING_Y)
2364                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2365         val |= I965_FENCE_REG_VALID;
2366
2367         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2368 }
2369
2370 static void i915_write_fence_reg(struct drm_gem_object *obj)
2371 {
2372         struct drm_device *dev = obj->dev;
2373         drm_i915_private_t *dev_priv = dev->dev_private;
2374         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2375         u32 size = i915_gem_get_gtt_size(obj_priv);
2376         uint32_t fence_reg, val, pitch_val;
2377         int tile_width;
2378
2379         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2380             (obj_priv->gtt_offset & (size - 1))) {
2381                 WARN(1, "%s: object 0x%08x [fenceable? %d] not 1M or size (0x%08x) aligned [gtt_space offset=%lx, size=%lx]\n",
2382                      __func__, obj_priv->gtt_offset, obj_priv->fenceable, size,
2383                      obj_priv->gtt_space->start, obj_priv->gtt_space->size);
2384                 return;
2385         }
2386
2387         if (obj_priv->tiling_mode == I915_TILING_Y &&
2388             HAS_128_BYTE_Y_TILING(dev))
2389                 tile_width = 128;
2390         else
2391                 tile_width = 512;
2392
2393         /* Note: pitch better be a power of two tile widths */
2394         pitch_val = obj_priv->stride / tile_width;
2395         pitch_val = ffs(pitch_val) - 1;
2396
2397         if (obj_priv->tiling_mode == I915_TILING_Y &&
2398             HAS_128_BYTE_Y_TILING(dev))
2399                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2400         else
2401                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2402
2403         val = obj_priv->gtt_offset;
2404         if (obj_priv->tiling_mode == I915_TILING_Y)
2405                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2406         val |= I915_FENCE_SIZE_BITS(size);
2407         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2408         val |= I830_FENCE_REG_VALID;
2409
2410         fence_reg = obj_priv->fence_reg;
2411         if (fence_reg < 8)
2412                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2413         else
2414                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2415         I915_WRITE(fence_reg, val);
2416 }
2417
2418 static void i830_write_fence_reg(struct drm_gem_object *obj)
2419 {
2420         struct drm_device *dev = obj->dev;
2421         drm_i915_private_t *dev_priv = dev->dev_private;
2422         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2423         u32 size = i915_gem_get_gtt_size(obj_priv);
2424         int regnum = obj_priv->fence_reg;
2425         uint32_t val;
2426         uint32_t pitch_val;
2427         uint32_t fence_size_bits;
2428
2429         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2430             (obj_priv->gtt_offset & (obj->size - 1))) {
2431                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2432                      __func__, obj_priv->gtt_offset);
2433                 return;
2434         }
2435
2436         pitch_val = obj_priv->stride / 128;
2437         pitch_val = ffs(pitch_val) - 1;
2438         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2439
2440         val = obj_priv->gtt_offset;
2441         if (obj_priv->tiling_mode == I915_TILING_Y)
2442                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2443         fence_size_bits = I830_FENCE_SIZE_BITS(size);
2444         WARN_ON(fence_size_bits & ~0x00000f00);
2445         val |= fence_size_bits;
2446         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2447         val |= I830_FENCE_REG_VALID;
2448
2449         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2450 }
2451
2452 static int i915_find_fence_reg(struct drm_device *dev,
2453                                bool interruptible)
2454 {
2455         struct drm_i915_private *dev_priv = dev->dev_private;
2456         struct drm_i915_fence_reg *reg;
2457         struct drm_i915_gem_object *obj_priv = NULL;
2458         int i, avail, ret;
2459
2460         /* First try to find a free reg */
2461         avail = 0;
2462         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2463                 reg = &dev_priv->fence_regs[i];
2464                 if (!reg->obj)
2465                         return i;
2466
2467                 obj_priv = to_intel_bo(reg->obj);
2468                 if (!obj_priv->pin_count)
2469                     avail++;
2470         }
2471
2472         if (avail == 0)
2473                 return -ENOSPC;
2474
2475         /* None available, try to steal one or wait for a user to finish */
2476         avail = I915_FENCE_REG_NONE;
2477         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2478                             lru_list) {
2479                 obj_priv = to_intel_bo(reg->obj);
2480                 if (obj_priv->pin_count)
2481                         continue;
2482
2483                 /* found one! */
2484                 avail = obj_priv->fence_reg;
2485                 break;
2486         }
2487
2488         BUG_ON(avail == I915_FENCE_REG_NONE);
2489
2490         /* We only have a reference on obj from the active list. put_fence_reg
2491          * might drop that one, causing a use-after-free in it. So hold a
2492          * private reference to obj like the other callers of put_fence_reg
2493          * (set_tiling ioctl) do. */
2494         drm_gem_object_reference(&obj_priv->base);
2495         ret = i915_gem_object_put_fence_reg(&obj_priv->base, interruptible);
2496         drm_gem_object_unreference(&obj_priv->base);
2497         if (ret != 0)
2498                 return ret;
2499
2500         return avail;
2501 }
2502
2503 /**
2504  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2505  * @obj: object to map through a fence reg
2506  *
2507  * When mapping objects through the GTT, userspace wants to be able to write
2508  * to them without having to worry about swizzling if the object is tiled.
2509  *
2510  * This function walks the fence regs looking for a free one for @obj,
2511  * stealing one if it can't find any.
2512  *
2513  * It then sets up the reg based on the object's properties: address, pitch
2514  * and tiling format.
2515  */
2516 int
2517 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2518                               bool interruptible)
2519 {
2520         struct drm_device *dev = obj->dev;
2521         struct drm_i915_private *dev_priv = dev->dev_private;
2522         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2523         struct drm_i915_fence_reg *reg = NULL;
2524         int ret;
2525
2526         /* Just update our place in the LRU if our fence is getting used. */
2527         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2528                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2529                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2530                 return 0;
2531         }
2532
2533         switch (obj_priv->tiling_mode) {
2534         case I915_TILING_NONE:
2535                 WARN(1, "allocating a fence for non-tiled object?\n");
2536                 break;
2537         case I915_TILING_X:
2538                 if (!obj_priv->stride)
2539                         return -EINVAL;
2540                 WARN((obj_priv->stride & (512 - 1)),
2541                      "object 0x%08x is X tiled but has non-512B pitch\n",
2542                      obj_priv->gtt_offset);
2543                 break;
2544         case I915_TILING_Y:
2545                 if (!obj_priv->stride)
2546                         return -EINVAL;
2547                 WARN((obj_priv->stride & (128 - 1)),
2548                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2549                      obj_priv->gtt_offset);
2550                 break;
2551         }
2552
2553         ret = i915_find_fence_reg(dev, interruptible);
2554         if (ret < 0)
2555                 return ret;
2556
2557         obj_priv->fence_reg = ret;
2558         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2559         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2560
2561         reg->obj = obj;
2562
2563         switch (INTEL_INFO(dev)->gen) {
2564         case 6:
2565                 sandybridge_write_fence_reg(obj);
2566                 break;
2567         case 5:
2568         case 4:
2569                 i965_write_fence_reg(obj);
2570                 break;
2571         case 3:
2572                 i915_write_fence_reg(obj);
2573                 break;
2574         case 2:
2575                 i830_write_fence_reg(obj);
2576                 break;
2577         }
2578
2579         trace_i915_gem_object_get_fence(obj,
2580                                         obj_priv->fence_reg,
2581                                         obj_priv->tiling_mode);
2582
2583         return 0;
2584 }
2585
2586 /**
2587  * i915_gem_clear_fence_reg - clear out fence register info
2588  * @obj: object to clear
2589  *
2590  * Zeroes out the fence register itself and clears out the associated
2591  * data structures in dev_priv and obj_priv.
2592  */
2593 static void
2594 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2595 {
2596         struct drm_device *dev = obj->dev;
2597         drm_i915_private_t *dev_priv = dev->dev_private;
2598         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2599         struct drm_i915_fence_reg *reg =
2600                 &dev_priv->fence_regs[obj_priv->fence_reg];
2601         uint32_t fence_reg;
2602
2603         switch (INTEL_INFO(dev)->gen) {
2604         case 6:
2605                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2606                              (obj_priv->fence_reg * 8), 0);
2607                 break;
2608         case 5:
2609         case 4:
2610                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2611                 break;
2612         case 3:
2613                 if (obj_priv->fence_reg >= 8)
2614                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2615                 else
2616         case 2:
2617                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2618
2619                 I915_WRITE(fence_reg, 0);
2620                 break;
2621         }
2622
2623         reg->obj = NULL;
2624         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2625         list_del_init(&reg->lru_list);
2626 }
2627
2628 /**
2629  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2630  * to the buffer to finish, and then resets the fence register.
2631  * @obj: tiled object holding a fence register.
2632  * @bool: whether the wait upon the fence is interruptible
2633  *
2634  * Zeroes out the fence register itself and clears out the associated
2635  * data structures in dev_priv and obj_priv.
2636  */
2637 int
2638 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2639                               bool interruptible)
2640 {
2641         struct drm_device *dev = obj->dev;
2642         struct drm_i915_private *dev_priv = dev->dev_private;
2643         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2644         struct drm_i915_fence_reg *reg;
2645
2646         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2647                 return 0;
2648
2649         /* If we've changed tiling, GTT-mappings of the object
2650          * need to re-fault to ensure that the correct fence register
2651          * setup is in place.
2652          */
2653         i915_gem_release_mmap(obj);
2654
2655         /* On the i915, GPU access to tiled buffers is via a fence,
2656          * therefore we must wait for any outstanding access to complete
2657          * before clearing the fence.
2658          */
2659         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2660         if (reg->gpu) {
2661                 int ret;
2662
2663                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2664                 if (ret)
2665                         return ret;
2666
2667                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2668                 if (ret)
2669                         return ret;
2670
2671                 reg->gpu = false;
2672         }
2673
2674         i915_gem_object_flush_gtt_write_domain(obj);
2675         i915_gem_clear_fence_reg(obj);
2676
2677         return 0;
2678 }
2679
2680 /**
2681  * Finds free space in the GTT aperture and binds the object there.
2682  */
2683 static int
2684 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
2685                             unsigned alignment,
2686                             bool mappable,
2687                             bool need_fence)
2688 {
2689         struct drm_device *dev = obj->dev;
2690         drm_i915_private_t *dev_priv = dev->dev_private;
2691         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2692         struct drm_mm_node *free_space;
2693         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2694         u32 size, fence_size, fence_alignment;
2695         int ret;
2696
2697         if (obj_priv->madv != I915_MADV_WILLNEED) {
2698                 DRM_ERROR("Attempting to bind a purgeable object\n");
2699                 return -EINVAL;
2700         }
2701
2702         fence_size = i915_gem_get_gtt_size(obj_priv);
2703         fence_alignment = i915_gem_get_gtt_alignment(obj_priv);
2704
2705         if (alignment == 0)
2706                 alignment = need_fence ? fence_alignment : 4096;
2707         if (need_fence && alignment & (fence_alignment - 1)) {
2708                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2709                 return -EINVAL;
2710         }
2711
2712         size = need_fence ? fence_size : obj->size;
2713
2714         /* If the object is bigger than the entire aperture, reject it early
2715          * before evicting everything in a vain attempt to find space.
2716          */
2717         if (obj->size >
2718             (mappable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2719                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2720                 return -E2BIG;
2721         }
2722
2723  search_free:
2724         if (mappable)
2725                 free_space =
2726                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2727                                                     size, alignment, 0,
2728                                                     dev_priv->mm.gtt_mappable_end,
2729                                                     0);
2730         else
2731                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2732                                                 size, alignment, 0);
2733
2734         if (free_space != NULL) {
2735                 if (mappable)
2736                         obj_priv->gtt_space =
2737                                 drm_mm_get_block_range_generic(free_space,
2738                                                                size, alignment, 0,
2739                                                                dev_priv->mm.gtt_mappable_end,
2740                                                                0);
2741                 else
2742                         obj_priv->gtt_space =
2743                                 drm_mm_get_block(free_space, size, alignment);
2744         }
2745         if (obj_priv->gtt_space == NULL) {
2746                 /* If the gtt is empty and we're still having trouble
2747                  * fitting our object in, we're out of memory.
2748                  */
2749                 ret = i915_gem_evict_something(dev, size, alignment, mappable);
2750                 if (ret)
2751                         return ret;
2752
2753                 goto search_free;
2754         }
2755
2756         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2757         if (ret) {
2758                 drm_mm_put_block(obj_priv->gtt_space);
2759                 obj_priv->gtt_space = NULL;
2760
2761                 if (ret == -ENOMEM) {
2762                         /* first try to clear up some space from the GTT */
2763                         ret = i915_gem_evict_something(dev, size,
2764                                                        alignment, mappable);
2765                         if (ret) {
2766                                 /* now try to shrink everyone else */
2767                                 if (gfpmask) {
2768                                         gfpmask = 0;
2769                                         goto search_free;
2770                                 }
2771
2772                                 return ret;
2773                         }
2774
2775                         goto search_free;
2776                 }
2777
2778                 return ret;
2779         }
2780
2781         /* Create an AGP memory structure pointing at our pages, and bind it
2782          * into the GTT.
2783          */
2784         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2785                                                obj_priv->pages,
2786                                                obj->size >> PAGE_SHIFT,
2787                                                obj_priv->gtt_space->start,
2788                                                obj_priv->agp_type);
2789         if (obj_priv->agp_mem == NULL) {
2790                 i915_gem_object_put_pages_gtt(obj);
2791                 drm_mm_put_block(obj_priv->gtt_space);
2792                 obj_priv->gtt_space = NULL;
2793
2794                 ret = i915_gem_evict_something(dev, size,
2795                                                alignment, mappable);
2796                 if (ret)
2797                         return ret;
2798
2799                 goto search_free;
2800         }
2801
2802         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2803
2804         /* keep track of bounds object by adding it to the inactive list */
2805         list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
2806         i915_gem_info_add_gtt(dev_priv, obj_priv);
2807
2808         /* Assert that the object is not currently in any GPU domain. As it
2809          * wasn't in the GTT, there shouldn't be any way it could have been in
2810          * a GPU cache
2811          */
2812         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2813         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2814
2815         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset, mappable);
2816
2817         obj_priv->fenceable =
2818                 obj_priv->gtt_space->size == fence_size &&
2819                 (obj_priv->gtt_space->start & (fence_alignment -1)) == 0;
2820
2821         obj_priv->mappable =
2822                 obj_priv->gtt_offset + obj->size <= dev_priv->mm.gtt_mappable_end;
2823
2824         return 0;
2825 }
2826
2827 void
2828 i915_gem_clflush_object(struct drm_gem_object *obj)
2829 {
2830         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2831
2832         /* If we don't have a page list set up, then we're not pinned
2833          * to GPU, and we can ignore the cache flush because it'll happen
2834          * again at bind time.
2835          */
2836         if (obj_priv->pages == NULL)
2837                 return;
2838
2839         trace_i915_gem_object_clflush(obj);
2840
2841         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2842 }
2843
2844 /** Flushes any GPU write domain for the object if it's dirty. */
2845 static int
2846 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2847                                        bool pipelined)
2848 {
2849         struct drm_device *dev = obj->dev;
2850
2851         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2852                 return 0;
2853
2854         /* Queue the GPU write cache flushing we need. */
2855         i915_gem_flush_ring(dev, NULL,
2856                             to_intel_bo(obj)->ring,
2857                             0, obj->write_domain);
2858         BUG_ON(obj->write_domain);
2859
2860         if (pipelined)
2861                 return 0;
2862
2863         return i915_gem_object_wait_rendering(obj, true);
2864 }
2865
2866 /** Flushes the GTT write domain for the object if it's dirty. */
2867 static void
2868 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2869 {
2870         uint32_t old_write_domain;
2871
2872         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2873                 return;
2874
2875         /* No actual flushing is required for the GTT write domain.   Writes
2876          * to it immediately go to main memory as far as we know, so there's
2877          * no chipset flush.  It also doesn't land in render cache.
2878          */
2879         i915_gem_release_mmap(obj);
2880
2881         old_write_domain = obj->write_domain;
2882         obj->write_domain = 0;
2883
2884         trace_i915_gem_object_change_domain(obj,
2885                                             obj->read_domains,
2886                                             old_write_domain);
2887 }
2888
2889 /** Flushes the CPU write domain for the object if it's dirty. */
2890 static void
2891 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2892 {
2893         struct drm_device *dev = obj->dev;
2894         uint32_t old_write_domain;
2895
2896         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2897                 return;
2898
2899         i915_gem_clflush_object(obj);
2900         drm_agp_chipset_flush(dev);
2901         old_write_domain = obj->write_domain;
2902         obj->write_domain = 0;
2903
2904         trace_i915_gem_object_change_domain(obj,
2905                                             obj->read_domains,
2906                                             old_write_domain);
2907 }
2908
2909 /**
2910  * Moves a single object to the GTT read, and possibly write domain.
2911  *
2912  * This function returns when the move is complete, including waiting on
2913  * flushes to occur.
2914  */
2915 int
2916 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2917 {
2918         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2919         uint32_t old_write_domain, old_read_domains;
2920         int ret;
2921
2922         /* Not valid to be called on unbound objects. */
2923         if (obj_priv->gtt_space == NULL)
2924                 return -EINVAL;
2925
2926         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2927         if (ret != 0)
2928                 return ret;
2929
2930         i915_gem_object_flush_cpu_write_domain(obj);
2931
2932         if (write) {
2933                 ret = i915_gem_object_wait_rendering(obj, true);
2934                 if (ret)
2935                         return ret;
2936         }
2937
2938         old_write_domain = obj->write_domain;
2939         old_read_domains = obj->read_domains;
2940
2941         /* It should now be out of any other write domains, and we can update
2942          * the domain values for our changes.
2943          */
2944         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2945         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2946         if (write) {
2947                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2948                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2949                 obj_priv->dirty = 1;
2950         }
2951
2952         trace_i915_gem_object_change_domain(obj,
2953                                             old_read_domains,
2954                                             old_write_domain);
2955
2956         return 0;
2957 }
2958
2959 /*
2960  * Prepare buffer for display plane. Use uninterruptible for possible flush
2961  * wait, as in modesetting process we're not supposed to be interrupted.
2962  */
2963 int
2964 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2965                                      bool pipelined)
2966 {
2967         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2968         uint32_t old_read_domains;
2969         int ret;
2970
2971         /* Not valid to be called on unbound objects. */
2972         if (obj_priv->gtt_space == NULL)
2973                 return -EINVAL;
2974
2975         ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2976         if (ret)
2977                 return ret;
2978
2979         /* Currently, we are always called from an non-interruptible context. */
2980         if (!pipelined) {
2981                 ret = i915_gem_object_wait_rendering(obj, false);
2982                 if (ret)
2983                         return ret;
2984         }
2985
2986         i915_gem_object_flush_cpu_write_domain(obj);
2987
2988         old_read_domains = obj->read_domains;
2989         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2990
2991         trace_i915_gem_object_change_domain(obj,
2992                                             old_read_domains,
2993                                             obj->write_domain);
2994
2995         return 0;
2996 }
2997
2998 /**
2999  * Moves a single object to the CPU read, and possibly write domain.
3000  *
3001  * This function returns when the move is complete, including waiting on
3002  * flushes to occur.
3003  */
3004 static int
3005 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
3006 {
3007         uint32_t old_write_domain, old_read_domains;
3008         int ret;
3009
3010         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3011         if (ret != 0)
3012                 return ret;
3013
3014         i915_gem_object_flush_gtt_write_domain(obj);
3015
3016         /* If we have a partially-valid cache of the object in the CPU,
3017          * finish invalidating it and free the per-page flags.
3018          */
3019         i915_gem_object_set_to_full_cpu_read_domain(obj);
3020
3021         if (write) {
3022                 ret = i915_gem_object_wait_rendering(obj, true);
3023                 if (ret)
3024                         return ret;
3025         }
3026
3027         old_write_domain = obj->write_domain;
3028         old_read_domains = obj->read_domains;
3029
3030         /* Flush the CPU cache if it's still invalid. */
3031         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3032                 i915_gem_clflush_object(obj);
3033
3034                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3035         }
3036
3037         /* It should now be out of any other write domains, and we can update
3038          * the domain values for our changes.
3039          */
3040         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3041
3042         /* If we're writing through the CPU, then the GPU read domains will
3043          * need to be invalidated at next use.
3044          */
3045         if (write) {
3046                 obj->read_domains = I915_GEM_DOMAIN_CPU;
3047                 obj->write_domain = I915_GEM_DOMAIN_CPU;
3048         }
3049
3050         trace_i915_gem_object_change_domain(obj,
3051                                             old_read_domains,
3052                                             old_write_domain);
3053
3054         return 0;
3055 }
3056
3057 /*
3058  * Set the next domain for the specified object. This
3059  * may not actually perform the necessary flushing/invaliding though,
3060  * as that may want to be batched with other set_domain operations
3061  *
3062  * This is (we hope) the only really tricky part of gem. The goal
3063  * is fairly simple -- track which caches hold bits of the object
3064  * and make sure they remain coherent. A few concrete examples may
3065  * help to explain how it works. For shorthand, we use the notation
3066  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3067  * a pair of read and write domain masks.
3068  *
3069  * Case 1: the batch buffer
3070  *
3071  *      1. Allocated
3072  *      2. Written by CPU
3073  *      3. Mapped to GTT
3074  *      4. Read by GPU
3075  *      5. Unmapped from GTT
3076  *      6. Freed
3077  *
3078  *      Let's take these a step at a time
3079  *
3080  *      1. Allocated
3081  *              Pages allocated from the kernel may still have
3082  *              cache contents, so we set them to (CPU, CPU) always.
3083  *      2. Written by CPU (using pwrite)
3084  *              The pwrite function calls set_domain (CPU, CPU) and
3085  *              this function does nothing (as nothing changes)
3086  *      3. Mapped by GTT
3087  *              This function asserts that the object is not
3088  *              currently in any GPU-based read or write domains
3089  *      4. Read by GPU
3090  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3091  *              As write_domain is zero, this function adds in the
3092  *              current read domains (CPU+COMMAND, 0).
3093  *              flush_domains is set to CPU.
3094  *              invalidate_domains is set to COMMAND
3095  *              clflush is run to get data out of the CPU caches
3096  *              then i915_dev_set_domain calls i915_gem_flush to
3097  *              emit an MI_FLUSH and drm_agp_chipset_flush
3098  *      5. Unmapped from GTT
3099  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3100  *              flush_domains and invalidate_domains end up both zero
3101  *              so no flushing/invalidating happens
3102  *      6. Freed
3103  *              yay, done
3104  *
3105  * Case 2: The shared render buffer
3106  *
3107  *      1. Allocated
3108  *      2. Mapped to GTT
3109  *      3. Read/written by GPU
3110  *      4. set_domain to (CPU,CPU)
3111  *      5. Read/written by CPU
3112  *      6. Read/written by GPU
3113  *
3114  *      1. Allocated
3115  *              Same as last example, (CPU, CPU)
3116  *      2. Mapped to GTT
3117  *              Nothing changes (assertions find that it is not in the GPU)
3118  *      3. Read/written by GPU
3119  *              execbuffer calls set_domain (RENDER, RENDER)
3120  *              flush_domains gets CPU
3121  *              invalidate_domains gets GPU
3122  *              clflush (obj)
3123  *              MI_FLUSH and drm_agp_chipset_flush
3124  *      4. set_domain (CPU, CPU)
3125  *              flush_domains gets GPU
3126  *              invalidate_domains gets CPU
3127  *              wait_rendering (obj) to make sure all drawing is complete.
3128  *              This will include an MI_FLUSH to get the data from GPU
3129  *              to memory
3130  *              clflush (obj) to invalidate the CPU cache
3131  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3132  *      5. Read/written by CPU
3133  *              cache lines are loaded and dirtied
3134  *      6. Read written by GPU
3135  *              Same as last GPU access
3136  *
3137  * Case 3: The constant buffer
3138  *
3139  *      1. Allocated
3140  *      2. Written by CPU
3141  *      3. Read by GPU
3142  *      4. Updated (written) by CPU again
3143  *      5. Read by GPU
3144  *
3145  *      1. Allocated
3146  *              (CPU, CPU)
3147  *      2. Written by CPU
3148  *              (CPU, CPU)
3149  *      3. Read by GPU
3150  *              (CPU+RENDER, 0)
3151  *              flush_domains = CPU
3152  *              invalidate_domains = RENDER
3153  *              clflush (obj)
3154  *              MI_FLUSH
3155  *              drm_agp_chipset_flush
3156  *      4. Updated (written) by CPU again
3157  *              (CPU, CPU)
3158  *              flush_domains = 0 (no previous write domain)
3159  *              invalidate_domains = 0 (no new read domains)
3160  *      5. Read by GPU
3161  *              (CPU+RENDER, 0)
3162  *              flush_domains = CPU
3163  *              invalidate_domains = RENDER
3164  *              clflush (obj)
3165  *              MI_FLUSH
3166  *              drm_agp_chipset_flush
3167  */
3168 static void
3169 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3170                                   struct intel_ring_buffer *ring)
3171 {
3172         struct drm_device               *dev = obj->dev;
3173         struct drm_i915_private         *dev_priv = dev->dev_private;
3174         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3175         uint32_t                        invalidate_domains = 0;
3176         uint32_t                        flush_domains = 0;
3177
3178         /*
3179          * If the object isn't moving to a new write domain,
3180          * let the object stay in multiple read domains
3181          */
3182         if (obj->pending_write_domain == 0)
3183                 obj->pending_read_domains |= obj->read_domains;
3184
3185         /*
3186          * Flush the current write domain if
3187          * the new read domains don't match. Invalidate
3188          * any read domains which differ from the old
3189          * write domain
3190          */
3191         if (obj->write_domain &&
3192             obj->write_domain != obj->pending_read_domains) {
3193                 flush_domains |= obj->write_domain;
3194                 invalidate_domains |=
3195                         obj->pending_read_domains & ~obj->write_domain;
3196         }
3197         /*
3198          * Invalidate any read caches which may have
3199          * stale data. That is, any new read domains.
3200          */
3201         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3202         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3203                 i915_gem_clflush_object(obj);
3204
3205         /* blow away mappings if mapped through GTT */
3206         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
3207                 i915_gem_release_mmap(obj);
3208
3209         /* The actual obj->write_domain will be updated with
3210          * pending_write_domain after we emit the accumulated flush for all
3211          * of our domain changes in execbuffers (which clears objects'
3212          * write_domains).  So if we have a current write domain that we
3213          * aren't changing, set pending_write_domain to that.
3214          */
3215         if (flush_domains == 0 && obj->pending_write_domain == 0)
3216                 obj->pending_write_domain = obj->write_domain;
3217
3218         dev->invalidate_domains |= invalidate_domains;
3219         dev->flush_domains |= flush_domains;
3220         if (flush_domains & I915_GEM_GPU_DOMAINS)
3221                 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3222         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3223                 dev_priv->mm.flush_rings |= ring->id;
3224 }
3225
3226 /**
3227  * Moves the object from a partially CPU read to a full one.
3228  *
3229  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3230  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3231  */
3232 static void
3233 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3234 {
3235         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3236
3237         if (!obj_priv->page_cpu_valid)
3238                 return;
3239
3240         /* If we're partially in the CPU read domain, finish moving it in.
3241          */
3242         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3243                 int i;
3244
3245                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3246                         if (obj_priv->page_cpu_valid[i])
3247                                 continue;
3248                         drm_clflush_pages(obj_priv->pages + i, 1);
3249                 }
3250         }
3251
3252         /* Free the page_cpu_valid mappings which are now stale, whether
3253          * or not we've got I915_GEM_DOMAIN_CPU.
3254          */
3255         kfree(obj_priv->page_cpu_valid);
3256         obj_priv->page_cpu_valid = NULL;
3257 }
3258
3259 /**
3260  * Set the CPU read domain on a range of the object.
3261  *
3262  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3263  * not entirely valid.  The page_cpu_valid member of the object flags which
3264  * pages have been flushed, and will be respected by
3265  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3266  * of the whole object.
3267  *
3268  * This function returns when the move is complete, including waiting on
3269  * flushes to occur.
3270  */
3271 static int
3272 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3273                                           uint64_t offset, uint64_t size)
3274 {
3275         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3276         uint32_t old_read_domains;
3277         int i, ret;
3278
3279         if (offset == 0 && size == obj->size)
3280                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3281
3282         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3283         if (ret != 0)
3284                 return ret;
3285         i915_gem_object_flush_gtt_write_domain(obj);
3286
3287         /* If we're already fully in the CPU read domain, we're done. */
3288         if (obj_priv->page_cpu_valid == NULL &&
3289             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3290                 return 0;
3291
3292         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3293          * newly adding I915_GEM_DOMAIN_CPU
3294          */
3295         if (obj_priv->page_cpu_valid == NULL) {
3296                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3297                                                    GFP_KERNEL);
3298                 if (obj_priv->page_cpu_valid == NULL)
3299                         return -ENOMEM;
3300         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3301                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3302
3303         /* Flush the cache on any pages that are still invalid from the CPU's
3304          * perspective.
3305          */
3306         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3307              i++) {
3308                 if (obj_priv->page_cpu_valid[i])
3309                         continue;
3310
3311                 drm_clflush_pages(obj_priv->pages + i, 1);
3312
3313                 obj_priv->page_cpu_valid[i] = 1;
3314         }
3315
3316         /* It should now be out of any other write domains, and we can update
3317          * the domain values for our changes.
3318          */
3319         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3320
3321         old_read_domains = obj->read_domains;
3322         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3323
3324         trace_i915_gem_object_change_domain(obj,
3325                                             old_read_domains,
3326                                             obj->write_domain);
3327
3328         return 0;
3329 }
3330
3331 /**
3332  * Pin an object to the GTT and evaluate the relocations landing in it.
3333  */
3334 static int
3335 i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
3336                              struct drm_file *file_priv,
3337                              struct drm_i915_gem_exec_object2 *entry)
3338 {
3339         struct drm_device *dev = obj->base.dev;
3340         drm_i915_private_t *dev_priv = dev->dev_private;
3341         struct drm_i915_gem_relocation_entry __user *user_relocs;
3342         struct drm_gem_object *target_obj = NULL;
3343         uint32_t target_handle = 0;
3344         int i, ret = 0;
3345
3346         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3347         for (i = 0; i < entry->relocation_count; i++) {
3348                 struct drm_i915_gem_relocation_entry reloc;
3349                 uint32_t target_offset;
3350
3351                 if (__copy_from_user_inatomic(&reloc,
3352                                               user_relocs+i,
3353                                               sizeof(reloc))) {
3354                         ret = -EFAULT;
3355                         break;
3356                 }
3357
3358                 if (reloc.target_handle != target_handle) {
3359                         drm_gem_object_unreference(target_obj);
3360
3361                         target_obj = drm_gem_object_lookup(dev, file_priv,
3362                                                            reloc.target_handle);
3363                         if (target_obj == NULL) {
3364                                 ret = -ENOENT;
3365                                 break;
3366                         }
3367
3368                         target_handle = reloc.target_handle;
3369                 }
3370                 target_offset = to_intel_bo(target_obj)->gtt_offset;
3371
3372 #if WATCH_RELOC
3373                 DRM_INFO("%s: obj %p offset %08x target %d "
3374                          "read %08x write %08x gtt %08x "
3375                          "presumed %08x delta %08x\n",
3376                          __func__,
3377                          obj,
3378                          (int) reloc.offset,
3379                          (int) reloc.target_handle,
3380                          (int) reloc.read_domains,
3381                          (int) reloc.write_domain,
3382                          (int) target_offset,
3383                          (int) reloc.presumed_offset,
3384                          reloc.delta);
3385 #endif
3386
3387                 /* The target buffer should have appeared before us in the
3388                  * exec_object list, so it should have a GTT space bound by now.
3389                  */
3390                 if (target_offset == 0) {
3391                         DRM_ERROR("No GTT space found for object %d\n",
3392                                   reloc.target_handle);
3393                         ret = -EINVAL;
3394                         break;
3395                 }
3396
3397                 /* Validate that the target is in a valid r/w GPU domain */
3398                 if (reloc.write_domain & (reloc.write_domain - 1)) {
3399                         DRM_ERROR("reloc with multiple write domains: "
3400                                   "obj %p target %d offset %d "
3401                                   "read %08x write %08x",
3402                                   obj, reloc.target_handle,
3403                                   (int) reloc.offset,
3404                                   reloc.read_domains,
3405                                   reloc.write_domain);
3406                         ret = -EINVAL;
3407                         break;
3408                 }
3409                 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3410                     reloc.read_domains & I915_GEM_DOMAIN_CPU) {
3411                         DRM_ERROR("reloc with read/write CPU domains: "
3412                                   "obj %p target %d offset %d "
3413                                   "read %08x write %08x",
3414                                   obj, reloc.target_handle,
3415                                   (int) reloc.offset,
3416                                   reloc.read_domains,
3417                                   reloc.write_domain);
3418                         ret = -EINVAL;
3419                         break;
3420                 }
3421                 if (reloc.write_domain && target_obj->pending_write_domain &&
3422                     reloc.write_domain != target_obj->pending_write_domain) {
3423                         DRM_ERROR("Write domain conflict: "
3424                                   "obj %p target %d offset %d "
3425                                   "new %08x old %08x\n",
3426                                   obj, reloc.target_handle,
3427                                   (int) reloc.offset,
3428                                   reloc.write_domain,
3429                                   target_obj->pending_write_domain);
3430                         ret = -EINVAL;
3431                         break;
3432                 }
3433
3434                 target_obj->pending_read_domains |= reloc.read_domains;
3435                 target_obj->pending_write_domain |= reloc.write_domain;
3436
3437                 /* If the relocation already has the right value in it, no
3438                  * more work needs to be done.
3439                  */
3440                 if (target_offset == reloc.presumed_offset)
3441                         continue;
3442
3443                 /* Check that the relocation address is valid... */
3444                 if (reloc.offset > obj->base.size - 4) {
3445                         DRM_ERROR("Relocation beyond object bounds: "
3446                                   "obj %p target %d offset %d size %d.\n",
3447                                   obj, reloc.target_handle,
3448                                   (int) reloc.offset, (int) obj->base.size);
3449                         ret = -EINVAL;
3450                         break;
3451                 }
3452                 if (reloc.offset & 3) {
3453                         DRM_ERROR("Relocation not 4-byte aligned: "
3454                                   "obj %p target %d offset %d.\n",
3455                                   obj, reloc.target_handle,
3456                                   (int) reloc.offset);
3457                         ret = -EINVAL;
3458                         break;
3459                 }
3460
3461                 /* and points to somewhere within the target object. */
3462                 if (reloc.delta >= target_obj->size) {
3463                         DRM_ERROR("Relocation beyond target object bounds: "
3464                                   "obj %p target %d delta %d size %d.\n",
3465                                   obj, reloc.target_handle,
3466                                   (int) reloc.delta, (int) target_obj->size);
3467                         ret = -EINVAL;
3468                         break;
3469                 }
3470
3471                 reloc.delta += target_offset;
3472                 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3473                         uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3474                         char *vaddr;
3475
3476                         vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT]);
3477                         *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3478                         kunmap_atomic(vaddr);
3479                 } else {
3480                         uint32_t __iomem *reloc_entry;
3481                         void __iomem *reloc_page;
3482
3483                         ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3484                         if (ret)
3485                                 break;
3486
3487                         /* Map the page containing the relocation we're going to perform.  */
3488                         reloc.offset += obj->gtt_offset;
3489                         reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3490                                                               reloc.offset & PAGE_MASK);
3491                         reloc_entry = (uint32_t __iomem *)
3492                                 (reloc_page + (reloc.offset & ~PAGE_MASK));
3493                         iowrite32(reloc.delta, reloc_entry);
3494                         io_mapping_unmap_atomic(reloc_page);
3495                 }
3496
3497                 /* and update the user's relocation entry */
3498                 reloc.presumed_offset = target_offset;
3499                 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3500                                               &reloc.presumed_offset,
3501                                               sizeof(reloc.presumed_offset))) {
3502                     ret = -EFAULT;
3503                     break;
3504                 }
3505         }
3506
3507         drm_gem_object_unreference(target_obj);
3508         return ret;
3509 }
3510
3511 static int
3512 i915_gem_execbuffer_pin(struct drm_device *dev,
3513                         struct drm_file *file,
3514                         struct drm_gem_object **object_list,
3515                         struct drm_i915_gem_exec_object2 *exec_list,
3516                         int count)
3517 {
3518         struct drm_i915_private *dev_priv = dev->dev_private;
3519         int ret, i, retry;
3520
3521         /* attempt to pin all of the buffers into the GTT */
3522         for (retry = 0; retry < 2; retry++) {
3523                 ret = 0;
3524                 for (i = 0; i < count; i++) {
3525                         struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3526                         struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3527                         bool need_fence =
3528                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3529                                 obj->tiling_mode != I915_TILING_NONE;
3530
3531                         /* g33/pnv can't fence buffers in the unmappable part */
3532                         bool need_mappable =
3533                                 entry->relocation_count ? true : need_fence;
3534
3535                         /* Check fence reg constraints and rebind if necessary */
3536                         if ((need_fence && !obj->fenceable) ||
3537                             (need_mappable && !obj->mappable)) {
3538                                 ret = i915_gem_object_unbind(&obj->base);
3539                                 if (ret)
3540                                         break;
3541                         }
3542
3543                         ret = i915_gem_object_pin(&obj->base,
3544                                                   entry->alignment,
3545                                                   need_mappable,
3546                                                   need_fence);
3547                         if (ret)
3548                                 break;
3549
3550                         /*
3551                          * Pre-965 chips need a fence register set up in order
3552                          * to properly handle blits to/from tiled surfaces.
3553                          */
3554                         if (need_fence) {
3555                                 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3556                                 if (ret) {
3557                                         i915_gem_object_unpin(&obj->base);
3558                                         break;
3559                                 }
3560
3561                                 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3562                         }
3563
3564                         entry->offset = obj->gtt_offset;
3565                 }
3566
3567                 while (i--)
3568                         i915_gem_object_unpin(object_list[i]);
3569
3570                 if (ret == 0)
3571                         break;
3572
3573                 if (ret != -ENOSPC || retry)
3574                         return ret;
3575
3576                 ret = i915_gem_evict_everything(dev);
3577                 if (ret)
3578                         return ret;
3579         }
3580
3581         return 0;
3582 }
3583
3584 /* Throttle our rendering by waiting until the ring has completed our requests
3585  * emitted over 20 msec ago.
3586  *
3587  * Note that if we were to use the current jiffies each time around the loop,
3588  * we wouldn't escape the function with any frames outstanding if the time to
3589  * render a frame was over 20ms.
3590  *
3591  * This should get us reasonable parallelism between CPU and GPU but also
3592  * relatively low latency when blocking on a particular request to finish.
3593  */
3594 static int
3595 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3596 {
3597         struct drm_i915_private *dev_priv = dev->dev_private;
3598         struct drm_i915_file_private *file_priv = file->driver_priv;
3599         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3600         struct drm_i915_gem_request *request;
3601         struct intel_ring_buffer *ring = NULL;
3602         u32 seqno = 0;
3603         int ret;
3604
3605         spin_lock(&file_priv->mm.lock);
3606         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3607                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3608                         break;
3609
3610                 ring = request->ring;
3611                 seqno = request->seqno;
3612         }
3613         spin_unlock(&file_priv->mm.lock);
3614
3615         if (seqno == 0)
3616                 return 0;
3617
3618         ret = 0;
3619         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3620                 /* And wait for the seqno passing without holding any locks and
3621                  * causing extra latency for others. This is safe as the irq
3622                  * generation is designed to be run atomically and so is
3623                  * lockless.
3624                  */
3625                 ring->user_irq_get(ring);
3626                 ret = wait_event_interruptible(ring->irq_queue,
3627                                                i915_seqno_passed(ring->get_seqno(ring), seqno)
3628                                                || atomic_read(&dev_priv->mm.wedged));
3629                 ring->user_irq_put(ring);
3630
3631                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3632                         ret = -EIO;
3633         }
3634
3635         if (ret == 0)
3636                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3637
3638         return ret;
3639 }
3640
3641 static int
3642 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3643                           uint64_t exec_offset)
3644 {
3645         uint32_t exec_start, exec_len;
3646
3647         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3648         exec_len = (uint32_t) exec->batch_len;
3649
3650         if ((exec_start | exec_len) & 0x7)
3651                 return -EINVAL;
3652
3653         if (!exec_start)
3654                 return -EINVAL;
3655
3656         return 0;
3657 }
3658
3659 static int
3660 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3661                    int count)
3662 {
3663         int i;
3664
3665         for (i = 0; i < count; i++) {
3666                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3667                 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
3668
3669                 if (!access_ok(VERIFY_READ, ptr, length))
3670                         return -EFAULT;
3671
3672                 /* we may also need to update the presumed offsets */
3673                 if (!access_ok(VERIFY_WRITE, ptr, length))
3674                         return -EFAULT;
3675
3676                 if (fault_in_pages_readable(ptr, length))
3677                         return -EFAULT;
3678         }
3679
3680         return 0;
3681 }
3682
3683 static int
3684 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3685                        struct drm_file *file,
3686                        struct drm_i915_gem_execbuffer2 *args,
3687                        struct drm_i915_gem_exec_object2 *exec_list)
3688 {
3689         drm_i915_private_t *dev_priv = dev->dev_private;
3690         struct drm_gem_object **object_list = NULL;
3691         struct drm_gem_object *batch_obj;
3692         struct drm_clip_rect *cliprects = NULL;
3693         struct drm_i915_gem_request *request = NULL;
3694         int ret, i, flips;
3695         uint64_t exec_offset;
3696
3697         struct intel_ring_buffer *ring = NULL;
3698
3699         ret = i915_gem_check_is_wedged(dev);
3700         if (ret)
3701                 return ret;
3702
3703         ret = validate_exec_list(exec_list, args->buffer_count);
3704         if (ret)
3705                 return ret;
3706
3707 #if WATCH_EXEC
3708         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3709                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3710 #endif
3711         switch (args->flags & I915_EXEC_RING_MASK) {
3712         case I915_EXEC_DEFAULT:
3713         case I915_EXEC_RENDER:
3714                 ring = &dev_priv->render_ring;
3715                 break;
3716         case I915_EXEC_BSD:
3717                 if (!HAS_BSD(dev)) {
3718                         DRM_ERROR("execbuf with invalid ring (BSD)\n");
3719                         return -EINVAL;
3720                 }
3721                 ring = &dev_priv->bsd_ring;
3722                 break;
3723         case I915_EXEC_BLT:
3724                 if (!HAS_BLT(dev)) {
3725                         DRM_ERROR("execbuf with invalid ring (BLT)\n");
3726                         return -EINVAL;
3727                 }
3728                 ring = &dev_priv->blt_ring;
3729                 break;
3730         default:
3731                 DRM_ERROR("execbuf with unknown ring: %d\n",
3732                           (int)(args->flags & I915_EXEC_RING_MASK));
3733                 return -EINVAL;
3734         }
3735
3736         if (args->buffer_count < 1) {
3737                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3738                 return -EINVAL;
3739         }
3740         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3741         if (object_list == NULL) {
3742                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3743                           args->buffer_count);
3744                 ret = -ENOMEM;
3745                 goto pre_mutex_err;
3746         }
3747
3748         if (args->num_cliprects != 0) {
3749                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3750                                     GFP_KERNEL);
3751                 if (cliprects == NULL) {
3752                         ret = -ENOMEM;
3753                         goto pre_mutex_err;
3754                 }
3755
3756                 ret = copy_from_user(cliprects,
3757                                      (struct drm_clip_rect __user *)
3758                                      (uintptr_t) args->cliprects_ptr,
3759                                      sizeof(*cliprects) * args->num_cliprects);
3760                 if (ret != 0) {
3761                         DRM_ERROR("copy %d cliprects failed: %d\n",
3762                                   args->num_cliprects, ret);
3763                         ret = -EFAULT;
3764                         goto pre_mutex_err;
3765                 }
3766         }
3767
3768         request = kzalloc(sizeof(*request), GFP_KERNEL);
3769         if (request == NULL) {
3770                 ret = -ENOMEM;
3771                 goto pre_mutex_err;
3772         }
3773
3774         ret = i915_mutex_lock_interruptible(dev);
3775         if (ret)
3776                 goto pre_mutex_err;
3777
3778         if (dev_priv->mm.suspended) {
3779                 mutex_unlock(&dev->struct_mutex);
3780                 ret = -EBUSY;
3781                 goto pre_mutex_err;
3782         }
3783
3784         /* Look up object handles */
3785         for (i = 0; i < args->buffer_count; i++) {
3786                 struct drm_i915_gem_object *obj_priv;
3787
3788                 object_list[i] = drm_gem_object_lookup(dev, file,
3789                                                        exec_list[i].handle);
3790                 if (object_list[i] == NULL) {
3791                         DRM_ERROR("Invalid object handle %d at index %d\n",
3792                                    exec_list[i].handle, i);
3793                         /* prevent error path from reading uninitialized data */
3794                         args->buffer_count = i + 1;
3795                         ret = -ENOENT;
3796                         goto err;
3797                 }
3798
3799                 obj_priv = to_intel_bo(object_list[i]);
3800                 if (obj_priv->in_execbuffer) {
3801                         DRM_ERROR("Object %p appears more than once in object list\n",
3802                                    object_list[i]);
3803                         /* prevent error path from reading uninitialized data */
3804                         args->buffer_count = i + 1;
3805                         ret = -EINVAL;
3806                         goto err;
3807                 }
3808                 obj_priv->in_execbuffer = true;
3809         }
3810
3811         /* Move the objects en-masse into the GTT, evicting if necessary. */
3812         ret = i915_gem_execbuffer_pin(dev, file,
3813                                       object_list, exec_list,
3814                                       args->buffer_count);
3815         if (ret)
3816                 goto err;
3817
3818         /* The objects are in their final locations, apply the relocations. */
3819         for (i = 0; i < args->buffer_count; i++) {
3820                 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3821                 obj->base.pending_read_domains = 0;
3822                 obj->base.pending_write_domain = 0;
3823                 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
3824                 if (ret)
3825                         goto err;
3826         }
3827
3828         /* Set the pending read domains for the batch buffer to COMMAND */
3829         batch_obj = object_list[args->buffer_count-1];
3830         if (batch_obj->pending_write_domain) {
3831                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3832                 ret = -EINVAL;
3833                 goto err;
3834         }
3835         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3836
3837         /* Sanity check the batch buffer */
3838         exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3839         ret = i915_gem_check_execbuffer(args, exec_offset);
3840         if (ret != 0) {
3841                 DRM_ERROR("execbuf with invalid offset/length\n");
3842                 goto err;
3843         }
3844
3845         /* Zero the global flush/invalidate flags. These
3846          * will be modified as new domains are computed
3847          * for each object
3848          */
3849         dev->invalidate_domains = 0;
3850         dev->flush_domains = 0;
3851         dev_priv->mm.flush_rings = 0;
3852         for (i = 0; i < args->buffer_count; i++)
3853                 i915_gem_object_set_to_gpu_domain(object_list[i], ring);
3854
3855         if (dev->invalidate_domains | dev->flush_domains) {
3856 #if WATCH_EXEC
3857                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3858                           __func__,
3859                          dev->invalidate_domains,
3860                          dev->flush_domains);
3861 #endif
3862                 i915_gem_flush(dev, file,
3863                                dev->invalidate_domains,
3864                                dev->flush_domains,
3865                                dev_priv->mm.flush_rings);
3866         }
3867
3868 #if WATCH_COHERENCY
3869         for (i = 0; i < args->buffer_count; i++) {
3870                 i915_gem_object_check_coherency(object_list[i],
3871                                                 exec_list[i].handle);
3872         }
3873 #endif
3874
3875 #if WATCH_EXEC
3876         i915_gem_dump_object(batch_obj,
3877                               args->batch_len,
3878                               __func__,
3879                               ~0);
3880 #endif
3881
3882         /* Check for any pending flips. As we only maintain a flip queue depth
3883          * of 1, we can simply insert a WAIT for the next display flip prior
3884          * to executing the batch and avoid stalling the CPU.
3885          */
3886         flips = 0;
3887         for (i = 0; i < args->buffer_count; i++) {
3888                 if (object_list[i]->write_domain)
3889                         flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3890         }
3891         if (flips) {
3892                 int plane, flip_mask;
3893
3894                 for (plane = 0; flips >> plane; plane++) {
3895                         if (((flips >> plane) & 1) == 0)
3896                                 continue;
3897
3898                         if (plane)
3899                                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3900                         else
3901                                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3902
3903                         ret = intel_ring_begin(ring, 2);
3904                         if (ret)
3905                                 goto err;
3906
3907                         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
3908                         intel_ring_emit(ring, MI_NOOP);
3909                         intel_ring_advance(ring);
3910                 }
3911         }
3912
3913         /* Exec the batchbuffer */
3914         ret = ring->dispatch_execbuffer(ring, args, cliprects, exec_offset);
3915         if (ret) {
3916                 DRM_ERROR("dispatch failed %d\n", ret);
3917                 goto err;
3918         }
3919
3920         for (i = 0; i < args->buffer_count; i++) {
3921                 struct drm_gem_object *obj = object_list[i];
3922
3923                 obj->read_domains = obj->pending_read_domains;
3924                 obj->write_domain = obj->pending_write_domain;
3925
3926                 i915_gem_object_move_to_active(obj, ring);
3927                 if (obj->write_domain) {
3928                         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3929                         obj_priv->dirty = 1;
3930                         list_move_tail(&obj_priv->gpu_write_list,
3931                                        &ring->gpu_write_list);
3932                         intel_mark_busy(dev, obj);
3933                 }
3934
3935                 trace_i915_gem_object_change_domain(obj,
3936                                                     obj->read_domains,
3937                                                     obj->write_domain);
3938         }
3939
3940         /*
3941          * Ensure that the commands in the batch buffer are
3942          * finished before the interrupt fires
3943          */
3944         i915_retire_commands(dev, ring);
3945
3946         if (i915_add_request(dev, file, request, ring))
3947                 ring->outstanding_lazy_request = true;
3948         else
3949                 request = NULL;
3950
3951 err:
3952         for (i = 0; i < args->buffer_count; i++) {
3953                 if (object_list[i] == NULL)
3954                     break;
3955
3956                 to_intel_bo(object_list[i])->in_execbuffer = false;
3957                 drm_gem_object_unreference(object_list[i]);
3958         }
3959
3960         mutex_unlock(&dev->struct_mutex);
3961
3962 pre_mutex_err:
3963         drm_free_large(object_list);
3964         kfree(cliprects);
3965         kfree(request);
3966
3967         return ret;
3968 }
3969
3970 /*
3971  * Legacy execbuffer just creates an exec2 list from the original exec object
3972  * list array and passes it to the real function.
3973  */
3974 int
3975 i915_gem_execbuffer(struct drm_device *dev, void *data,
3976                     struct drm_file *file_priv)
3977 {
3978         struct drm_i915_gem_execbuffer *args = data;
3979         struct drm_i915_gem_execbuffer2 exec2;
3980         struct drm_i915_gem_exec_object *exec_list = NULL;
3981         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3982         int ret, i;
3983
3984 #if WATCH_EXEC
3985         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3986                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3987 #endif
3988
3989         if (args->buffer_count < 1) {
3990                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3991                 return -EINVAL;
3992         }
3993
3994         /* Copy in the exec list from userland */
3995         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3996         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3997         if (exec_list == NULL || exec2_list == NULL) {
3998                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3999                           args->buffer_count);
4000                 drm_free_large(exec_list);
4001                 drm_free_large(exec2_list);
4002                 return -ENOMEM;
4003         }
4004         ret = copy_from_user(exec_list,
4005                              (struct drm_i915_relocation_entry __user *)
4006                              (uintptr_t) args->buffers_ptr,
4007                              sizeof(*exec_list) * args->buffer_count);
4008         if (ret != 0) {
4009                 DRM_ERROR("copy %d exec entries failed %d\n",
4010                           args->buffer_count, ret);
4011                 drm_free_large(exec_list);
4012                 drm_free_large(exec2_list);
4013                 return -EFAULT;
4014         }
4015
4016         for (i = 0; i < args->buffer_count; i++) {
4017                 exec2_list[i].handle = exec_list[i].handle;
4018                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4019                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4020                 exec2_list[i].alignment = exec_list[i].alignment;
4021                 exec2_list[i].offset = exec_list[i].offset;
4022                 if (INTEL_INFO(dev)->gen < 4)
4023                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4024                 else
4025                         exec2_list[i].flags = 0;
4026         }
4027
4028         exec2.buffers_ptr = args->buffers_ptr;
4029         exec2.buffer_count = args->buffer_count;
4030         exec2.batch_start_offset = args->batch_start_offset;
4031         exec2.batch_len = args->batch_len;
4032         exec2.DR1 = args->DR1;
4033         exec2.DR4 = args->DR4;
4034         exec2.num_cliprects = args->num_cliprects;
4035         exec2.cliprects_ptr = args->cliprects_ptr;
4036         exec2.flags = I915_EXEC_RENDER;
4037
4038         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4039         if (!ret) {
4040                 /* Copy the new buffer offsets back to the user's exec list. */
4041                 for (i = 0; i < args->buffer_count; i++)
4042                         exec_list[i].offset = exec2_list[i].offset;
4043                 /* ... and back out to userspace */
4044                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4045                                    (uintptr_t) args->buffers_ptr,
4046                                    exec_list,
4047                                    sizeof(*exec_list) * args->buffer_count);
4048                 if (ret) {
4049                         ret = -EFAULT;
4050                         DRM_ERROR("failed to copy %d exec entries "
4051                                   "back to user (%d)\n",
4052                                   args->buffer_count, ret);
4053                 }
4054         }
4055
4056         drm_free_large(exec_list);
4057         drm_free_large(exec2_list);
4058         return ret;
4059 }
4060
4061 int
4062 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4063                      struct drm_file *file_priv)
4064 {
4065         struct drm_i915_gem_execbuffer2 *args = data;
4066         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4067         int ret;
4068
4069 #if WATCH_EXEC
4070         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4071                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4072 #endif
4073
4074         if (args->buffer_count < 1) {
4075                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4076                 return -EINVAL;
4077         }
4078
4079         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4080         if (exec2_list == NULL) {
4081                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4082                           args->buffer_count);
4083                 return -ENOMEM;
4084         }
4085         ret = copy_from_user(exec2_list,
4086                              (struct drm_i915_relocation_entry __user *)
4087                              (uintptr_t) args->buffers_ptr,
4088                              sizeof(*exec2_list) * args->buffer_count);
4089         if (ret != 0) {
4090                 DRM_ERROR("copy %d exec entries failed %d\n",
4091                           args->buffer_count, ret);
4092                 drm_free_large(exec2_list);
4093                 return -EFAULT;
4094         }
4095
4096         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4097         if (!ret) {
4098                 /* Copy the new buffer offsets back to the user's exec list. */
4099                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4100                                    (uintptr_t) args->buffers_ptr,
4101                                    exec2_list,
4102                                    sizeof(*exec2_list) * args->buffer_count);
4103                 if (ret) {
4104                         ret = -EFAULT;
4105                         DRM_ERROR("failed to copy %d exec entries "
4106                                   "back to user (%d)\n",
4107                                   args->buffer_count, ret);
4108                 }
4109         }
4110
4111         drm_free_large(exec2_list);
4112         return ret;
4113 }
4114
4115 int
4116 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment,
4117                     bool mappable, bool need_fence)
4118 {
4119         struct drm_device *dev = obj->dev;
4120         struct drm_i915_private *dev_priv = dev->dev_private;
4121         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4122         int ret;
4123
4124         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4125         WARN_ON(i915_verify_lists(dev));
4126
4127         if (obj_priv->gtt_space != NULL) {
4128                 if ((alignment && obj_priv->gtt_offset & (alignment - 1)) ||
4129                     (need_fence && !obj_priv->fenceable) ||
4130                     (mappable && !obj_priv->mappable)) {
4131                         WARN(obj_priv->pin_count,
4132                              "bo is already pinned with incorrect alignment:"
4133                              " offset=%x, req.alignment=%x, need_fence=%d, fenceable=%d, mappable=%d, cpu_accessible=%d\n",
4134                              obj_priv->gtt_offset, alignment,
4135                              need_fence, obj_priv->fenceable,
4136                              mappable, obj_priv->mappable);
4137                         ret = i915_gem_object_unbind(obj);
4138                         if (ret)
4139                                 return ret;
4140                 }
4141         }
4142
4143         if (obj_priv->gtt_space == NULL) {
4144                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
4145                                                   mappable, need_fence);
4146                 if (ret)
4147                         return ret;
4148         }
4149
4150         if (obj_priv->pin_count++ == 0) {
4151                 i915_gem_info_add_pin(dev_priv, obj_priv, mappable);
4152                 if (!obj_priv->active)
4153                         list_move_tail(&obj_priv->mm_list,
4154                                        &dev_priv->mm.pinned_list);
4155         }
4156         BUG_ON(!obj_priv->pin_mappable && mappable);
4157
4158         WARN_ON(i915_verify_lists(dev));
4159         return 0;
4160 }
4161
4162 void
4163 i915_gem_object_unpin(struct drm_gem_object *obj)
4164 {
4165         struct drm_device *dev = obj->dev;
4166         drm_i915_private_t *dev_priv = dev->dev_private;
4167         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4168
4169         WARN_ON(i915_verify_lists(dev));
4170         BUG_ON(obj_priv->pin_count == 0);
4171         BUG_ON(obj_priv->gtt_space == NULL);
4172
4173         if (--obj_priv->pin_count == 0) {
4174                 if (!obj_priv->active)
4175                         list_move_tail(&obj_priv->mm_list,
4176                                        &dev_priv->mm.inactive_list);
4177                 i915_gem_info_remove_pin(dev_priv, obj_priv);
4178         }
4179         WARN_ON(i915_verify_lists(dev));
4180 }
4181
4182 int
4183 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4184                    struct drm_file *file_priv)
4185 {
4186         struct drm_i915_gem_pin *args = data;
4187         struct drm_gem_object *obj;
4188         struct drm_i915_gem_object *obj_priv;
4189         int ret;
4190
4191         ret = i915_mutex_lock_interruptible(dev);
4192         if (ret)
4193                 return ret;
4194
4195         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4196         if (obj == NULL) {
4197                 ret = -ENOENT;
4198                 goto unlock;
4199         }
4200         obj_priv = to_intel_bo(obj);
4201
4202         if (obj_priv->madv != I915_MADV_WILLNEED) {
4203                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4204                 ret = -EINVAL;
4205                 goto out;
4206         }
4207
4208         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4209                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4210                           args->handle);
4211                 ret = -EINVAL;
4212                 goto out;
4213         }
4214
4215         obj_priv->user_pin_count++;
4216         obj_priv->pin_filp = file_priv;
4217         if (obj_priv->user_pin_count == 1) {
4218                 ret = i915_gem_object_pin(obj, args->alignment,
4219                                           true, obj_priv->tiling_mode);
4220                 if (ret)
4221                         goto out;
4222         }
4223
4224         /* XXX - flush the CPU caches for pinned objects
4225          * as the X server doesn't manage domains yet
4226          */
4227         i915_gem_object_flush_cpu_write_domain(obj);
4228         args->offset = obj_priv->gtt_offset;
4229 out:
4230         drm_gem_object_unreference(obj);
4231 unlock:
4232         mutex_unlock(&dev->struct_mutex);
4233         return ret;
4234 }
4235
4236 int
4237 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4238                      struct drm_file *file_priv)
4239 {
4240         struct drm_i915_gem_pin *args = data;
4241         struct drm_gem_object *obj;
4242         struct drm_i915_gem_object *obj_priv;
4243         int ret;
4244
4245         ret = i915_mutex_lock_interruptible(dev);
4246         if (ret)
4247                 return ret;
4248
4249         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4250         if (obj == NULL) {
4251                 ret = -ENOENT;
4252                 goto unlock;
4253         }
4254         obj_priv = to_intel_bo(obj);
4255
4256         if (obj_priv->pin_filp != file_priv) {
4257                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4258                           args->handle);
4259                 ret = -EINVAL;
4260                 goto out;
4261         }
4262         obj_priv->user_pin_count--;
4263         if (obj_priv->user_pin_count == 0) {
4264                 obj_priv->pin_filp = NULL;
4265                 i915_gem_object_unpin(obj);
4266         }
4267
4268 out:
4269         drm_gem_object_unreference(obj);
4270 unlock:
4271         mutex_unlock(&dev->struct_mutex);
4272         return ret;
4273 }
4274
4275 int
4276 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4277                     struct drm_file *file_priv)
4278 {
4279         struct drm_i915_gem_busy *args = data;
4280         struct drm_gem_object *obj;
4281         struct drm_i915_gem_object *obj_priv;
4282         int ret;
4283
4284         ret = i915_mutex_lock_interruptible(dev);
4285         if (ret)
4286                 return ret;
4287
4288         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4289         if (obj == NULL) {
4290                 ret = -ENOENT;
4291                 goto unlock;
4292         }
4293         obj_priv = to_intel_bo(obj);
4294
4295         /* Count all active objects as busy, even if they are currently not used
4296          * by the gpu. Users of this interface expect objects to eventually
4297          * become non-busy without any further actions, therefore emit any
4298          * necessary flushes here.
4299          */
4300         args->busy = obj_priv->active;
4301         if (args->busy) {
4302                 /* Unconditionally flush objects, even when the gpu still uses this
4303                  * object. Userspace calling this function indicates that it wants to
4304                  * use this buffer rather sooner than later, so issuing the required
4305                  * flush earlier is beneficial.
4306                  */
4307                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4308                         i915_gem_flush_ring(dev, file_priv,
4309                                             obj_priv->ring,
4310                                             0, obj->write_domain);
4311
4312                 /* Update the active list for the hardware's current position.
4313                  * Otherwise this only updates on a delayed timer or when irqs
4314                  * are actually unmasked, and our working set ends up being
4315                  * larger than required.
4316                  */
4317                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4318
4319                 args->busy = obj_priv->active;
4320         }
4321
4322         drm_gem_object_unreference(obj);
4323 unlock:
4324         mutex_unlock(&dev->struct_mutex);
4325         return ret;
4326 }
4327
4328 int
4329 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4330                         struct drm_file *file_priv)
4331 {
4332     return i915_gem_ring_throttle(dev, file_priv);
4333 }
4334
4335 int
4336 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4337                        struct drm_file *file_priv)
4338 {
4339         struct drm_i915_gem_madvise *args = data;
4340         struct drm_gem_object *obj;
4341         struct drm_i915_gem_object *obj_priv;
4342         int ret;
4343
4344         switch (args->madv) {
4345         case I915_MADV_DONTNEED:
4346         case I915_MADV_WILLNEED:
4347             break;
4348         default:
4349             return -EINVAL;
4350         }
4351
4352         ret = i915_mutex_lock_interruptible(dev);
4353         if (ret)
4354                 return ret;
4355
4356         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4357         if (obj == NULL) {
4358                 ret = -ENOENT;
4359                 goto unlock;
4360         }
4361         obj_priv = to_intel_bo(obj);
4362
4363         if (obj_priv->pin_count) {
4364                 ret = -EINVAL;
4365                 goto out;
4366         }
4367
4368         if (obj_priv->madv != __I915_MADV_PURGED)
4369                 obj_priv->madv = args->madv;
4370
4371         /* if the object is no longer bound, discard its backing storage */
4372         if (i915_gem_object_is_purgeable(obj_priv) &&
4373             obj_priv->gtt_space == NULL)
4374                 i915_gem_object_truncate(obj);
4375
4376         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4377
4378 out:
4379         drm_gem_object_unreference(obj);
4380 unlock:
4381         mutex_unlock(&dev->struct_mutex);
4382         return ret;
4383 }
4384
4385 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4386                                               size_t size)
4387 {
4388         struct drm_i915_private *dev_priv = dev->dev_private;
4389         struct drm_i915_gem_object *obj;
4390
4391         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4392         if (obj == NULL)
4393                 return NULL;
4394
4395         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4396                 kfree(obj);
4397                 return NULL;
4398         }
4399
4400         i915_gem_info_add_obj(dev_priv, size);
4401
4402         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4403         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4404
4405         obj->agp_type = AGP_USER_MEMORY;
4406         obj->base.driver_private = NULL;
4407         obj->fence_reg = I915_FENCE_REG_NONE;
4408         INIT_LIST_HEAD(&obj->mm_list);
4409         INIT_LIST_HEAD(&obj->ring_list);
4410         INIT_LIST_HEAD(&obj->gpu_write_list);
4411         obj->madv = I915_MADV_WILLNEED;
4412         obj->fenceable = true;
4413         obj->mappable = true;
4414
4415         return &obj->base;
4416 }
4417
4418 int i915_gem_init_object(struct drm_gem_object *obj)
4419 {
4420         BUG();
4421
4422         return 0;
4423 }
4424
4425 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4426 {
4427         struct drm_device *dev = obj->dev;
4428         drm_i915_private_t *dev_priv = dev->dev_private;
4429         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4430         int ret;
4431
4432         ret = i915_gem_object_unbind(obj);
4433         if (ret == -ERESTARTSYS) {
4434                 list_move(&obj_priv->mm_list,
4435                           &dev_priv->mm.deferred_free_list);
4436                 return;
4437         }
4438
4439         if (obj->map_list.map)
4440                 i915_gem_free_mmap_offset(obj);
4441
4442         drm_gem_object_release(obj);
4443         i915_gem_info_remove_obj(dev_priv, obj->size);
4444
4445         kfree(obj_priv->page_cpu_valid);
4446         kfree(obj_priv->bit_17);
4447         kfree(obj_priv);
4448 }
4449
4450 void i915_gem_free_object(struct drm_gem_object *obj)
4451 {
4452         struct drm_device *dev = obj->dev;
4453         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4454
4455         trace_i915_gem_object_destroy(obj);
4456
4457         while (obj_priv->pin_count > 0)
4458                 i915_gem_object_unpin(obj);
4459
4460         if (obj_priv->phys_obj)
4461                 i915_gem_detach_phys_object(dev, obj);
4462
4463         i915_gem_free_object_tail(obj);
4464 }
4465
4466 int
4467 i915_gem_idle(struct drm_device *dev)
4468 {
4469         drm_i915_private_t *dev_priv = dev->dev_private;
4470         int ret;
4471
4472         mutex_lock(&dev->struct_mutex);
4473
4474         if (dev_priv->mm.suspended) {
4475                 mutex_unlock(&dev->struct_mutex);
4476                 return 0;
4477         }
4478
4479         ret = i915_gpu_idle(dev);
4480         if (ret) {
4481                 mutex_unlock(&dev->struct_mutex);
4482                 return ret;
4483         }
4484
4485         /* Under UMS, be paranoid and evict. */
4486         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4487                 ret = i915_gem_evict_inactive(dev);
4488                 if (ret) {
4489                         mutex_unlock(&dev->struct_mutex);
4490                         return ret;
4491                 }
4492         }
4493
4494         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4495          * We need to replace this with a semaphore, or something.
4496          * And not confound mm.suspended!
4497          */
4498         dev_priv->mm.suspended = 1;
4499         del_timer_sync(&dev_priv->hangcheck_timer);
4500
4501         i915_kernel_lost_context(dev);
4502         i915_gem_cleanup_ringbuffer(dev);
4503
4504         mutex_unlock(&dev->struct_mutex);
4505
4506         /* Cancel the retire work handler, which should be idle now. */
4507         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4508
4509         return 0;
4510 }
4511
4512 /*
4513  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4514  * over cache flushing.
4515  */
4516 static int
4517 i915_gem_init_pipe_control(struct drm_device *dev)
4518 {
4519         drm_i915_private_t *dev_priv = dev->dev_private;
4520         struct drm_gem_object *obj;
4521         struct drm_i915_gem_object *obj_priv;
4522         int ret;
4523
4524         obj = i915_gem_alloc_object(dev, 4096);
4525         if (obj == NULL) {
4526                 DRM_ERROR("Failed to allocate seqno page\n");
4527                 ret = -ENOMEM;
4528                 goto err;
4529         }
4530         obj_priv = to_intel_bo(obj);
4531         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4532
4533         ret = i915_gem_object_pin(obj, 4096, true, false);
4534         if (ret)
4535                 goto err_unref;
4536
4537         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4538         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4539         if (dev_priv->seqno_page == NULL)
4540                 goto err_unpin;
4541
4542         dev_priv->seqno_obj = obj;
4543         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4544
4545         return 0;
4546
4547 err_unpin:
4548         i915_gem_object_unpin(obj);
4549 err_unref:
4550         drm_gem_object_unreference(obj);
4551 err:
4552         return ret;
4553 }
4554
4555
4556 static void
4557 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4558 {
4559         drm_i915_private_t *dev_priv = dev->dev_private;
4560         struct drm_gem_object *obj;
4561         struct drm_i915_gem_object *obj_priv;
4562
4563         obj = dev_priv->seqno_obj;
4564         obj_priv = to_intel_bo(obj);
4565         kunmap(obj_priv->pages[0]);
4566         i915_gem_object_unpin(obj);
4567         drm_gem_object_unreference(obj);
4568         dev_priv->seqno_obj = NULL;
4569
4570         dev_priv->seqno_page = NULL;
4571 }
4572
4573 int
4574 i915_gem_init_ringbuffer(struct drm_device *dev)
4575 {
4576         drm_i915_private_t *dev_priv = dev->dev_private;
4577         int ret;
4578
4579         if (HAS_PIPE_CONTROL(dev)) {
4580                 ret = i915_gem_init_pipe_control(dev);
4581                 if (ret)
4582                         return ret;
4583         }
4584
4585         ret = intel_init_render_ring_buffer(dev);
4586         if (ret)
4587                 goto cleanup_pipe_control;
4588
4589         if (HAS_BSD(dev)) {
4590                 ret = intel_init_bsd_ring_buffer(dev);
4591                 if (ret)
4592                         goto cleanup_render_ring;
4593         }
4594
4595         if (HAS_BLT(dev)) {
4596                 ret = intel_init_blt_ring_buffer(dev);
4597                 if (ret)
4598                         goto cleanup_bsd_ring;
4599         }
4600
4601         dev_priv->next_seqno = 1;
4602
4603         return 0;
4604
4605 cleanup_bsd_ring:
4606         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4607 cleanup_render_ring:
4608         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4609 cleanup_pipe_control:
4610         if (HAS_PIPE_CONTROL(dev))
4611                 i915_gem_cleanup_pipe_control(dev);
4612         return ret;
4613 }
4614
4615 void
4616 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4617 {
4618         drm_i915_private_t *dev_priv = dev->dev_private;
4619
4620         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4621         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4622         intel_cleanup_ring_buffer(&dev_priv->blt_ring);
4623         if (HAS_PIPE_CONTROL(dev))
4624                 i915_gem_cleanup_pipe_control(dev);
4625 }
4626
4627 int
4628 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4629                        struct drm_file *file_priv)
4630 {
4631         drm_i915_private_t *dev_priv = dev->dev_private;
4632         int ret;
4633
4634         if (drm_core_check_feature(dev, DRIVER_MODESET))
4635                 return 0;
4636
4637         if (atomic_read(&dev_priv->mm.wedged)) {
4638                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4639                 atomic_set(&dev_priv->mm.wedged, 0);
4640         }
4641
4642         mutex_lock(&dev->struct_mutex);
4643         dev_priv->mm.suspended = 0;
4644
4645         ret = i915_gem_init_ringbuffer(dev);
4646         if (ret != 0) {
4647                 mutex_unlock(&dev->struct_mutex);
4648                 return ret;
4649         }
4650
4651         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4652         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4653         BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4654         BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4655         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4656         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4657         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4658         BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4659         BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4660         mutex_unlock(&dev->struct_mutex);
4661
4662         ret = drm_irq_install(dev);
4663         if (ret)
4664                 goto cleanup_ringbuffer;
4665
4666         return 0;
4667
4668 cleanup_ringbuffer:
4669         mutex_lock(&dev->struct_mutex);
4670         i915_gem_cleanup_ringbuffer(dev);
4671         dev_priv->mm.suspended = 1;
4672         mutex_unlock(&dev->struct_mutex);
4673
4674         return ret;
4675 }
4676
4677 int
4678 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4679                        struct drm_file *file_priv)
4680 {
4681         if (drm_core_check_feature(dev, DRIVER_MODESET))
4682                 return 0;
4683
4684         drm_irq_uninstall(dev);
4685         return i915_gem_idle(dev);
4686 }
4687
4688 void
4689 i915_gem_lastclose(struct drm_device *dev)
4690 {
4691         int ret;
4692
4693         if (drm_core_check_feature(dev, DRIVER_MODESET))
4694                 return;
4695
4696         ret = i915_gem_idle(dev);
4697         if (ret)
4698                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4699 }
4700
4701 static void
4702 init_ring_lists(struct intel_ring_buffer *ring)
4703 {
4704         INIT_LIST_HEAD(&ring->active_list);
4705         INIT_LIST_HEAD(&ring->request_list);
4706         INIT_LIST_HEAD(&ring->gpu_write_list);
4707 }
4708
4709 void
4710 i915_gem_load(struct drm_device *dev)
4711 {
4712         int i;
4713         drm_i915_private_t *dev_priv = dev->dev_private;
4714
4715         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4716         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4717         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4718         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4719         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4720         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4721         init_ring_lists(&dev_priv->render_ring);
4722         init_ring_lists(&dev_priv->bsd_ring);
4723         init_ring_lists(&dev_priv->blt_ring);
4724         for (i = 0; i < 16; i++)
4725                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4726         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4727                           i915_gem_retire_work_handler);
4728         init_completion(&dev_priv->error_completion);
4729
4730         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4731         if (IS_GEN3(dev)) {
4732                 u32 tmp = I915_READ(MI_ARB_STATE);
4733                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4734                         /* arb state is a masked write, so set bit + bit in mask */
4735                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4736                         I915_WRITE(MI_ARB_STATE, tmp);
4737                 }
4738         }
4739
4740         /* Old X drivers will take 0-2 for front, back, depth buffers */
4741         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4742                 dev_priv->fence_reg_start = 3;
4743
4744         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4745                 dev_priv->num_fence_regs = 16;
4746         else
4747                 dev_priv->num_fence_regs = 8;
4748
4749         /* Initialize fence registers to zero */
4750         switch (INTEL_INFO(dev)->gen) {
4751         case 6:
4752                 for (i = 0; i < 16; i++)
4753                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4754                 break;
4755         case 5:
4756         case 4:
4757                 for (i = 0; i < 16; i++)
4758                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4759                 break;
4760         case 3:
4761                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4762                         for (i = 0; i < 8; i++)
4763                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4764         case 2:
4765                 for (i = 0; i < 8; i++)
4766                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4767                 break;
4768         }
4769         i915_gem_detect_bit_6_swizzle(dev);
4770         init_waitqueue_head(&dev_priv->pending_flip_queue);
4771
4772         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4773         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4774         register_shrinker(&dev_priv->mm.inactive_shrinker);
4775 }
4776
4777 /*
4778  * Create a physically contiguous memory object for this object
4779  * e.g. for cursor + overlay regs
4780  */
4781 static int i915_gem_init_phys_object(struct drm_device *dev,
4782                                      int id, int size, int align)
4783 {
4784         drm_i915_private_t *dev_priv = dev->dev_private;
4785         struct drm_i915_gem_phys_object *phys_obj;
4786         int ret;
4787
4788         if (dev_priv->mm.phys_objs[id - 1] || !size)
4789                 return 0;
4790
4791         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4792         if (!phys_obj)
4793                 return -ENOMEM;
4794
4795         phys_obj->id = id;
4796
4797         phys_obj->handle = drm_pci_alloc(dev, size, align);
4798         if (!phys_obj->handle) {
4799                 ret = -ENOMEM;
4800                 goto kfree_obj;
4801         }
4802 #ifdef CONFIG_X86
4803         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4804 #endif
4805
4806         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4807
4808         return 0;
4809 kfree_obj:
4810         kfree(phys_obj);
4811         return ret;
4812 }
4813
4814 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4815 {
4816         drm_i915_private_t *dev_priv = dev->dev_private;
4817         struct drm_i915_gem_phys_object *phys_obj;
4818
4819         if (!dev_priv->mm.phys_objs[id - 1])
4820                 return;
4821
4822         phys_obj = dev_priv->mm.phys_objs[id - 1];
4823         if (phys_obj->cur_obj) {
4824                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4825         }
4826
4827 #ifdef CONFIG_X86
4828         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4829 #endif
4830         drm_pci_free(dev, phys_obj->handle);
4831         kfree(phys_obj);
4832         dev_priv->mm.phys_objs[id - 1] = NULL;
4833 }
4834
4835 void i915_gem_free_all_phys_object(struct drm_device *dev)
4836 {
4837         int i;
4838
4839         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4840                 i915_gem_free_phys_object(dev, i);
4841 }
4842
4843 void i915_gem_detach_phys_object(struct drm_device *dev,
4844                                  struct drm_gem_object *obj)
4845 {
4846         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4847         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4848         char *vaddr;
4849         int i;
4850         int page_count;
4851
4852         if (!obj_priv->phys_obj)
4853                 return;
4854         vaddr = obj_priv->phys_obj->handle->vaddr;
4855
4856         page_count = obj->size / PAGE_SIZE;
4857
4858         for (i = 0; i < page_count; i++) {
4859                 struct page *page = read_cache_page_gfp(mapping, i,
4860                                                         GFP_HIGHUSER | __GFP_RECLAIMABLE);
4861                 if (!IS_ERR(page)) {
4862                         char *dst = kmap_atomic(page);
4863                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4864                         kunmap_atomic(dst);
4865
4866                         drm_clflush_pages(&page, 1);
4867
4868                         set_page_dirty(page);
4869                         mark_page_accessed(page);
4870                         page_cache_release(page);
4871                 }
4872         }
4873         drm_agp_chipset_flush(dev);
4874
4875         obj_priv->phys_obj->cur_obj = NULL;
4876         obj_priv->phys_obj = NULL;
4877 }
4878
4879 int
4880 i915_gem_attach_phys_object(struct drm_device *dev,
4881                             struct drm_gem_object *obj,
4882                             int id,
4883                             int align)
4884 {
4885         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4886         drm_i915_private_t *dev_priv = dev->dev_private;
4887         struct drm_i915_gem_object *obj_priv;
4888         int ret = 0;
4889         int page_count;
4890         int i;
4891
4892         if (id > I915_MAX_PHYS_OBJECT)
4893                 return -EINVAL;
4894
4895         obj_priv = to_intel_bo(obj);
4896
4897         if (obj_priv->phys_obj) {
4898                 if (obj_priv->phys_obj->id == id)
4899                         return 0;
4900                 i915_gem_detach_phys_object(dev, obj);
4901         }
4902
4903         /* create a new object */
4904         if (!dev_priv->mm.phys_objs[id - 1]) {
4905                 ret = i915_gem_init_phys_object(dev, id,
4906                                                 obj->size, align);
4907                 if (ret) {
4908                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4909                         return ret;
4910                 }
4911         }
4912
4913         /* bind to the object */
4914         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4915         obj_priv->phys_obj->cur_obj = obj;
4916
4917         page_count = obj->size / PAGE_SIZE;
4918
4919         for (i = 0; i < page_count; i++) {
4920                 struct page *page;
4921                 char *dst, *src;
4922
4923                 page = read_cache_page_gfp(mapping, i,
4924                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
4925                 if (IS_ERR(page))
4926                         return PTR_ERR(page);
4927
4928                 src = kmap_atomic(obj_priv->pages[i]);
4929                 dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4930                 memcpy(dst, src, PAGE_SIZE);
4931                 kunmap_atomic(src);
4932
4933                 mark_page_accessed(page);
4934                 page_cache_release(page);
4935         }
4936
4937         return 0;
4938 }
4939
4940 static int
4941 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4942                      struct drm_i915_gem_pwrite *args,
4943                      struct drm_file *file_priv)
4944 {
4945         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4946         void *obj_addr;
4947         int ret;
4948         char __user *user_data;
4949
4950         user_data = (char __user *) (uintptr_t) args->data_ptr;
4951         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4952
4953         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4954         ret = copy_from_user(obj_addr, user_data, args->size);
4955         if (ret)
4956                 return -EFAULT;
4957
4958         drm_agp_chipset_flush(dev);
4959         return 0;
4960 }
4961
4962 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4963 {
4964         struct drm_i915_file_private *file_priv = file->driver_priv;
4965
4966         /* Clean up our request list when the client is going away, so that
4967          * later retire_requests won't dereference our soon-to-be-gone
4968          * file_priv.
4969          */
4970         spin_lock(&file_priv->mm.lock);
4971         while (!list_empty(&file_priv->mm.request_list)) {
4972                 struct drm_i915_gem_request *request;
4973
4974                 request = list_first_entry(&file_priv->mm.request_list,
4975                                            struct drm_i915_gem_request,
4976                                            client_list);
4977                 list_del(&request->client_list);
4978                 request->file_priv = NULL;
4979         }
4980         spin_unlock(&file_priv->mm.lock);
4981 }
4982
4983 static int
4984 i915_gpu_is_active(struct drm_device *dev)
4985 {
4986         drm_i915_private_t *dev_priv = dev->dev_private;
4987         int lists_empty;
4988
4989         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4990                       list_empty(&dev_priv->mm.active_list);
4991
4992         return !lists_empty;
4993 }
4994
4995 static int
4996 i915_gem_inactive_shrink(struct shrinker *shrinker,
4997                          int nr_to_scan,
4998                          gfp_t gfp_mask)
4999 {
5000         struct drm_i915_private *dev_priv =
5001                 container_of(shrinker,
5002                              struct drm_i915_private,
5003                              mm.inactive_shrinker);
5004         struct drm_device *dev = dev_priv->dev;
5005         struct drm_i915_gem_object *obj, *next;
5006         int cnt;
5007
5008         if (!mutex_trylock(&dev->struct_mutex))
5009                 return 0;
5010
5011         /* "fast-path" to count number of available objects */
5012         if (nr_to_scan == 0) {
5013                 cnt = 0;
5014                 list_for_each_entry(obj,
5015                                     &dev_priv->mm.inactive_list,
5016                                     mm_list)
5017                         cnt++;
5018                 mutex_unlock(&dev->struct_mutex);
5019                 return cnt / 100 * sysctl_vfs_cache_pressure;
5020         }
5021
5022 rescan:
5023         /* first scan for clean buffers */
5024         i915_gem_retire_requests(dev);
5025
5026         list_for_each_entry_safe(obj, next,
5027                                  &dev_priv->mm.inactive_list,
5028                                  mm_list) {
5029                 if (i915_gem_object_is_purgeable(obj)) {
5030                         i915_gem_object_unbind(&obj->base);
5031                         if (--nr_to_scan == 0)
5032                                 break;
5033                 }
5034         }
5035
5036         /* second pass, evict/count anything still on the inactive list */
5037         cnt = 0;
5038         list_for_each_entry_safe(obj, next,
5039                                  &dev_priv->mm.inactive_list,
5040                                  mm_list) {
5041                 if (nr_to_scan) {
5042                         i915_gem_object_unbind(&obj->base);
5043                         nr_to_scan--;
5044                 } else
5045                         cnt++;
5046         }
5047
5048         if (nr_to_scan && i915_gpu_is_active(dev)) {
5049                 /*
5050                  * We are desperate for pages, so as a last resort, wait
5051                  * for the GPU to finish and discard whatever we can.
5052                  * This has a dramatic impact to reduce the number of
5053                  * OOM-killer events whilst running the GPU aggressively.
5054                  */
5055                 if (i915_gpu_idle(dev) == 0)
5056                         goto rescan;
5057         }
5058         mutex_unlock(&dev->struct_mutex);
5059         return cnt / 100 * sysctl_vfs_cache_pressure;
5060 }