drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/shmem_fs.h>
  35 #include <linux/slab.h>
  36 #include <linux/swap.h>
  37 #include <linux/pci.h>
  38
  39 static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  42 static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  43                                                     unsigned alignment,
  44                                                     bool map_and_fenceable);
  45 static void i915_gem_clear_fence_reg(struct drm_device *dev,
  46                                      struct drm_i915_fence_reg *reg);
  47 static int i915_gem_phys_pwrite(struct drm_device *dev,
  48                                 struct drm_i915_gem_object *obj,
  49                                 struct drm_i915_gem_pwrite *args,
  50                                 struct drm_file *file);
  51 static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
  52
  53 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  54                                     struct shrink_control *sc);
  55 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  56
  57 /* some bookkeeping */
  58 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  59                                   size_t size)
  60 {
  61         dev_priv->mm.object_count++;
  62         dev_priv->mm.object_memory += size;
  63 }
  64
  65 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  66                                      size_t size)
  67 {
  68         dev_priv->mm.object_count--;
  69         dev_priv->mm.object_memory -= size;
  70 }
  71
  72 static int
  73 i915_gem_wait_for_error(struct drm_device *dev)
  74 {
  75         struct drm_i915_private *dev_priv = dev->dev_private;
  76         struct completion *x = &dev_priv->error_completion;
  77         unsigned long flags;
  78         int ret;
  79
  80         if (!atomic_read(&dev_priv->mm.wedged))
  81                 return 0;
  82
  83         ret = wait_for_completion_interruptible(x);
  84         if (ret)
  85                 return ret;
  86
  87         if (atomic_read(&dev_priv->mm.wedged)) {
  88                 /* GPU is hung, bump the completion count to account for
  89                  * the token we just consumed so that we never hit zero and
  90                  * end up waiting upon a subsequent completion event that
  91                  * will never happen.
  92                  */
  93                 spin_lock_irqsave(&x->wait.lock, flags);
  94                 x->done++;
  95                 spin_unlock_irqrestore(&x->wait.lock, flags);
  96         }
  97         return 0;
  98 }
  99
 100 int i915_mutex_lock_interruptible(struct drm_device *dev)
 101 {
 102         int ret;
 103
 104         ret = i915_gem_wait_for_error(dev);
 105         if (ret)
 106                 return ret;
 107
 108         ret = mutex_lock_interruptible(&dev->struct_mutex);
 109         if (ret)
 110                 return ret;
 111
 112         WARN_ON(i915_verify_lists(dev));
 113         return 0;
 114 }
 115
 116 static inline bool
 117 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 118 {
 119         return obj->gtt_space && !obj->active && obj->pin_count == 0;
 120 }
 121
 122 int
 123 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 124                     struct drm_file *file)
 125 {
 126         struct drm_i915_gem_init *args = data;
 127
 128         if (args->gtt_start >= args->gtt_end ||
 129             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 130                 return -EINVAL;
 131
 132         /* GEM with user mode setting was never supported on ilk and later. */
 133         if (INTEL_INFO(dev)->gen >= 5)
 134                 return -ENODEV;
 135
 136         mutex_lock(&dev->struct_mutex);
 137         i915_gem_init_global_gtt(dev, args->gtt_start,
 138                                  args->gtt_end, args->gtt_end);
 139         mutex_unlock(&dev->struct_mutex);
 140
 141         return 0;
 142 }
 143
 144 int
 145 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 146                             struct drm_file *file)
 147 {
 148         struct drm_i915_private *dev_priv = dev->dev_private;
 149         struct drm_i915_gem_get_aperture *args = data;
 150         struct drm_i915_gem_object *obj;
 151         size_t pinned;
 152
 153         if (!(dev->driver->driver_features & DRIVER_GEM))
 154                 return -ENODEV;
 155
 156         pinned = 0;
 157         mutex_lock(&dev->struct_mutex);
 158         list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
 159                 pinned += obj->gtt_space->size;
 160         mutex_unlock(&dev->struct_mutex);
 161
 162         args->aper_size = dev_priv->mm.gtt_total;
 163         args->aper_available_size = args->aper_size - pinned;
 164
 165         return 0;
 166 }
 167
 168 static int
 169 i915_gem_create(struct drm_file *file,
 170                 struct drm_device *dev,
 171                 uint64_t size,
 172                 uint32_t *handle_p)
 173 {
 174         struct drm_i915_gem_object *obj;
 175         int ret;
 176         u32 handle;
 177
 178         size = roundup(size, PAGE_SIZE);
 179         if (size == 0)
 180                 return -EINVAL;
 181
 182         /* Allocate the new object */
 183         obj = i915_gem_alloc_object(dev, size);
 184         if (obj == NULL)
 185                 return -ENOMEM;
 186
 187         ret = drm_gem_handle_create(file, &obj->base, &handle);
 188         if (ret) {
 189                 drm_gem_object_release(&obj->base);
 190                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 191                 kfree(obj);
 192                 return ret;
 193         }
 194
 195         /* drop reference from allocate - handle holds it now */
 196         drm_gem_object_unreference(&obj->base);
 197         trace_i915_gem_object_create(obj);
 198
 199         *handle_p = handle;
 200         return 0;
 201 }
 202
 203 int
 204 i915_gem_dumb_create(struct drm_file *file,
 205                      struct drm_device *dev,
 206                      struct drm_mode_create_dumb *args)
 207 {
 208         /* have to work out size/pitch and return them */
 209         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
 210         args->size = args->pitch * args->height;
 211         return i915_gem_create(file, dev,
 212                                args->size, &args->handle);
 213 }
 214
 215 int i915_gem_dumb_destroy(struct drm_file *file,
 216                           struct drm_device *dev,
 217                           uint32_t handle)
 218 {
 219         return drm_gem_handle_delete(file, handle);
 220 }
 221
 222 /**
 223  * Creates a new mm object and returns a handle to it.
 224  */
 225 int
 226 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 227                       struct drm_file *file)
 228 {
 229         struct drm_i915_gem_create *args = data;
 230         return i915_gem_create(file, dev,
 231                                args->size, &args->handle);
 232 }
 233
 234 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 235 {
 236         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
 237
 238         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 239                 obj->tiling_mode != I915_TILING_NONE;
 240 }
 241
 242 static inline int
 243 __copy_to_user_swizzled(char __user *cpu_vaddr,
 244                         const char *gpu_vaddr, int gpu_offset,
 245                         int length)
 246 {
 247         int ret, cpu_offset = 0;
 248
 249         while (length > 0) {
 250                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 251                 int this_length = min(cacheline_end - gpu_offset, length);
 252                 int swizzled_gpu_offset = gpu_offset ^ 64;
 253
 254                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 255                                      gpu_vaddr + swizzled_gpu_offset,
 256                                      this_length);
 257                 if (ret)
 258                         return ret + length;
 259
 260                 cpu_offset += this_length;
 261                 gpu_offset += this_length;
 262                 length -= this_length;
 263         }
 264
 265         return 0;
 266 }
 267
 268 static inline int
 269 __copy_from_user_swizzled(char __user *gpu_vaddr, int gpu_offset,
 270                           const char *cpu_vaddr,
 271                           int length)
 272 {
 273         int ret, cpu_offset = 0;
 274
 275         while (length > 0) {
 276                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 277                 int this_length = min(cacheline_end - gpu_offset, length);
 278                 int swizzled_gpu_offset = gpu_offset ^ 64;
 279
 280                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 281                                        cpu_vaddr + cpu_offset,
 282                                        this_length);
 283                 if (ret)
 284                         return ret + length;
 285
 286                 cpu_offset += this_length;
 287                 gpu_offset += this_length;
 288                 length -= this_length;
 289         }
 290
 291         return 0;
 292 }
 293
 294 /* Per-page copy function for the shmem pread fastpath.
 295  * Flushes invalid cachelines before reading the target if
 296  * needs_clflush is set. */
 297 static int
 298 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 299                  char __user *user_data,
 300                  bool page_do_bit17_swizzling, bool needs_clflush)
 301 {
 302         char *vaddr;
 303         int ret;
 304
 305         if (unlikely(page_do_bit17_swizzling))
 306                 return -EINVAL;
 307
 308         vaddr = kmap_atomic(page);
 309         if (needs_clflush)
 310                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 311                                        page_length);
 312         ret = __copy_to_user_inatomic(user_data,
 313                                       vaddr + shmem_page_offset,
 314                                       page_length);
 315         kunmap_atomic(vaddr);
 316
 317         return ret;
 318 }
 319
 320 static void
 321 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 322                              bool swizzled)
 323 {
 324         if (unlikely(swizzled)) {
 325                 unsigned long start = (unsigned long) addr;
 326                 unsigned long end = (unsigned long) addr + length;
 327
 328                 /* For swizzling simply ensure that we always flush both
 329                  * channels. Lame, but simple and it works. Swizzled
 330                  * pwrite/pread is far from a hotpath - current userspace
 331                  * doesn't use it at all. */
 332                 start = round_down(start, 128);
 333                 end = round_up(end, 128);
 334
 335                 drm_clflush_virt_range((void *)start, end - start);
 336         } else {
 337                 drm_clflush_virt_range(addr, length);
 338         }
 339
 340 }
 341
 342 /* Only difference to the fast-path function is that this can handle bit17
 343  * and uses non-atomic copy and kmap functions. */
 344 static int
 345 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 346                  char __user *user_data,
 347                  bool page_do_bit17_swizzling, bool needs_clflush)
 348 {
 349         char *vaddr;
 350         int ret;
 351
 352         vaddr = kmap(page);
 353         if (needs_clflush)
 354                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 355                                              page_length,
 356                                              page_do_bit17_swizzling);
 357
 358         if (page_do_bit17_swizzling)
 359                 ret = __copy_to_user_swizzled(user_data,
 360                                               vaddr, shmem_page_offset,
 361                                               page_length);
 362         else
 363                 ret = __copy_to_user(user_data,
 364                                      vaddr + shmem_page_offset,
 365                                      page_length);
 366         kunmap(page);
 367
 368         return ret;
 369 }
 370
 371 static int
 372 i915_gem_shmem_pread(struct drm_device *dev,
 373                      struct drm_i915_gem_object *obj,
 374                      struct drm_i915_gem_pread *args,
 375                      struct drm_file *file)
 376 {
 377         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 378         char __user *user_data;
 379         ssize_t remain;
 380         loff_t offset;
 381         int shmem_page_offset, page_length, ret = 0;
 382         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 383         int hit_slowpath = 0;
 384         int prefaulted = 0;
 385         int needs_clflush = 0;
 386         int release_page;
 387
 388         user_data = (char __user *) (uintptr_t) args->data_ptr;
 389         remain = args->size;
 390
 391         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 392
 393         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 394                 /* If we're not in the cpu read domain, set ourself into the gtt
 395                  * read domain and manually flush cachelines (if required). This
 396                  * optimizes for the case when the gpu will dirty the data
 397                  * anyway again before the next pread happens. */
 398                 if (obj->cache_level == I915_CACHE_NONE)
 399                         needs_clflush = 1;
 400                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 401                 if (ret)
 402                         return ret;
 403         }
 404
 405         offset = args->offset;
 406
 407         while (remain > 0) {
 408                 struct page *page;
 409
 410                 /* Operation in this page
 411                  *
 412                  * shmem_page_offset = offset within page in shmem file
 413                  * page_length = bytes to copy for this page
 414                  */
 415                 shmem_page_offset = offset_in_page(offset);
 416                 page_length = remain;
 417                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 418                         page_length = PAGE_SIZE - shmem_page_offset;
 419
 420                 if (obj->pages) {
 421                         page = obj->pages[offset >> PAGE_SHIFT];
 422                         release_page = 0;
 423                 } else {
 424                         page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
 425                         if (IS_ERR(page)) {
 426                                 ret = PTR_ERR(page);
 427                                 goto out;
 428                         }
 429                         release_page = 1;
 430                 }
 431
 432                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 433                         (page_to_phys(page) & (1 << 17)) != 0;
 434
 435                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 436                                        user_data, page_do_bit17_swizzling,
 437                                        needs_clflush);
 438                 if (ret == 0)
 439                         goto next_page;
 440
 441                 hit_slowpath = 1;
 442                 page_cache_get(page);
 443                 mutex_unlock(&dev->struct_mutex);
 444
 445                 if (!prefaulted) {
 446                         ret = fault_in_multipages_writeable(user_data, remain);
 447                         /* Userspace is tricking us, but we've already clobbered
 448                          * its pages with the prefault and promised to write the
 449                          * data up to the first fault. Hence ignore any errors
 450                          * and just continue. */
 451                         (void)ret;
 452                         prefaulted = 1;
 453                 }
 454
 455                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 456                                        user_data, page_do_bit17_swizzling,
 457                                        needs_clflush);
 458
 459                 mutex_lock(&dev->struct_mutex);
 460                 page_cache_release(page);
 461 next_page:
 462                 mark_page_accessed(page);
 463                 if (release_page)
 464                         page_cache_release(page);
 465
 466                 if (ret) {
 467                         ret = -EFAULT;
 468                         goto out;
 469                 }
 470
 471                 remain -= page_length;
 472                 user_data += page_length;
 473                 offset += page_length;
 474         }
 475
 476 out:
 477         if (hit_slowpath) {
 478                 /* Fixup: Kill any reinstated backing storage pages */
 479                 if (obj->madv == __I915_MADV_PURGED)
 480                         i915_gem_object_truncate(obj);
 481         }
 482
 483         return ret;
 484 }
 485
 486 /**
 487  * Reads data from the object referenced by handle.
 488  *
 489  * On error, the contents of *data are undefined.
 490  */
 491 int
 492 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 493                      struct drm_file *file)
 494 {
 495         struct drm_i915_gem_pread *args = data;
 496         struct drm_i915_gem_object *obj;
 497         int ret = 0;
 498
 499         if (args->size == 0)
 500                 return 0;
 501
 502         if (!access_ok(VERIFY_WRITE,
 503                        (char __user *)(uintptr_t)args->data_ptr,
 504                        args->size))
 505                 return -EFAULT;
 506
 507         ret = i915_mutex_lock_interruptible(dev);
 508         if (ret)
 509                 return ret;
 510
 511         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 512         if (&obj->base == NULL) {
 513                 ret = -ENOENT;
 514                 goto unlock;
 515         }
 516
 517         /* Bounds check source.  */
 518         if (args->offset > obj->base.size ||
 519             args->size > obj->base.size - args->offset) {
 520                 ret = -EINVAL;
 521                 goto out;
 522         }
 523
 524         trace_i915_gem_object_pread(obj, args->offset, args->size);
 525
 526         ret = i915_gem_shmem_pread(dev, obj, args, file);
 527
 528 out:
 529         drm_gem_object_unreference(&obj->base);
 530 unlock:
 531         mutex_unlock(&dev->struct_mutex);
 532         return ret;
 533 }
 534
 535 /* This is the fast write path which cannot handle
 536  * page faults in the source data
 537  */
 538
 539 static inline int
 540 fast_user_write(struct io_mapping *mapping,
 541                 loff_t page_base, int page_offset,
 542                 char __user *user_data,
 543                 int length)
 544 {
 545         char *vaddr_atomic;
 546         unsigned long unwritten;
 547
 548         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 549         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 550                                                       user_data, length);
 551         io_mapping_unmap_atomic(vaddr_atomic);
 552         return unwritten;
 553 }
 554
 555 /**
 556  * This is the fast pwrite path, where we copy the data directly from the
 557  * user into the GTT, uncached.
 558  */
 559 static int
 560 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 561                          struct drm_i915_gem_object *obj,
 562                          struct drm_i915_gem_pwrite *args,
 563                          struct drm_file *file)
 564 {
 565         drm_i915_private_t *dev_priv = dev->dev_private;
 566         ssize_t remain;
 567         loff_t offset, page_base;
 568         char __user *user_data;
 569         int page_offset, page_length, ret;
 570
 571         ret = i915_gem_object_pin(obj, 0, true);
 572         if (ret)
 573                 goto out;
 574
 575         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 576         if (ret)
 577                 goto out_unpin;
 578
 579         ret = i915_gem_object_put_fence(obj);
 580         if (ret)
 581                 goto out_unpin;
 582
 583         user_data = (char __user *) (uintptr_t) args->data_ptr;
 584         remain = args->size;
 585
 586         offset = obj->gtt_offset + args->offset;
 587
 588         while (remain > 0) {
 589                 /* Operation in this page
 590                  *
 591                  * page_base = page offset within aperture
 592                  * page_offset = offset within page
 593                  * page_length = bytes to copy for this page
 594                  */
 595                 page_base = offset & PAGE_MASK;
 596                 page_offset = offset_in_page(offset);
 597                 page_length = remain;
 598                 if ((page_offset + remain) > PAGE_SIZE)
 599                         page_length = PAGE_SIZE - page_offset;
 600
 601                 /* If we get a fault while copying data, then (presumably) our
 602                  * source page isn't available.  Return the error and we'll
 603                  * retry in the slow path.
 604                  */
 605                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 606                                     page_offset, user_data, page_length)) {
 607                         ret = -EFAULT;
 608                         goto out_unpin;
 609                 }
 610
 611                 remain -= page_length;
 612                 user_data += page_length;
 613                 offset += page_length;
 614         }
 615
 616 out_unpin:
 617         i915_gem_object_unpin(obj);
 618 out:
 619         return ret;
 620 }
 621
 622 /* Per-page copy function for the shmem pwrite fastpath.
 623  * Flushes invalid cachelines before writing to the target if
 624  * needs_clflush_before is set and flushes out any written cachelines after
 625  * writing if needs_clflush is set. */
 626 static int
 627 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 628                   char __user *user_data,
 629                   bool page_do_bit17_swizzling,
 630                   bool needs_clflush_before,
 631                   bool needs_clflush_after)
 632 {
 633         char *vaddr;
 634         int ret;
 635
 636         if (unlikely(page_do_bit17_swizzling))
 637                 return -EINVAL;
 638
 639         vaddr = kmap_atomic(page);
 640         if (needs_clflush_before)
 641                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 642                                        page_length);
 643         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
 644                                                 user_data,
 645                                                 page_length);
 646         if (needs_clflush_after)
 647                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 648                                        page_length);
 649         kunmap_atomic(vaddr);
 650
 651         return ret;
 652 }
 653
 654 /* Only difference to the fast-path function is that this can handle bit17
 655  * and uses non-atomic copy and kmap functions. */
 656 static int
 657 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 658                   char __user *user_data,
 659                   bool page_do_bit17_swizzling,
 660                   bool needs_clflush_before,
 661                   bool needs_clflush_after)
 662 {
 663         char *vaddr;
 664         int ret;
 665
 666         vaddr = kmap(page);
 667         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 668                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 669                                              page_length,
 670                                              page_do_bit17_swizzling);
 671         if (page_do_bit17_swizzling)
 672                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 673                                                 user_data,
 674                                                 page_length);
 675         else
 676                 ret = __copy_from_user(vaddr + shmem_page_offset,
 677                                        user_data,
 678                                        page_length);
 679         if (needs_clflush_after)
 680                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 681                                              page_length,
 682                                              page_do_bit17_swizzling);
 683         kunmap(page);
 684
 685         return ret;
 686 }
 687
 688 static int
 689 i915_gem_shmem_pwrite(struct drm_device *dev,
 690                       struct drm_i915_gem_object *obj,
 691                       struct drm_i915_gem_pwrite *args,
 692                       struct drm_file *file)
 693 {
 694         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 695         ssize_t remain;
 696         loff_t offset;
 697         char __user *user_data;
 698         int shmem_page_offset, page_length, ret = 0;
 699         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 700         int hit_slowpath = 0;
 701         int needs_clflush_after = 0;
 702         int needs_clflush_before = 0;
 703         int release_page;
 704
 705         user_data = (char __user *) (uintptr_t) args->data_ptr;
 706         remain = args->size;
 707
 708         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 709
 710         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 711                 /* If we're not in the cpu write domain, set ourself into the gtt
 712                  * write domain and manually flush cachelines (if required). This
 713                  * optimizes for the case when the gpu will use the data
 714                  * right away and we therefore have to clflush anyway. */
 715                 if (obj->cache_level == I915_CACHE_NONE)
 716                         needs_clflush_after = 1;
 717                 ret = i915_gem_object_set_to_gtt_domain(obj, true);
 718                 if (ret)
 719                         return ret;
 720         }
 721         /* Same trick applies for invalidate partially written cachelines before
 722          * writing.  */
 723         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
 724             && obj->cache_level == I915_CACHE_NONE)
 725                 needs_clflush_before = 1;
 726
 727         offset = args->offset;
 728         obj->dirty = 1;
 729
 730         while (remain > 0) {
 731                 struct page *page;
 732                 int partial_cacheline_write;
 733
 734                 /* Operation in this page
 735                  *
 736                  * shmem_page_offset = offset within page in shmem file
 737                  * page_length = bytes to copy for this page
 738                  */
 739                 shmem_page_offset = offset_in_page(offset);
 740
 741                 page_length = remain;
 742                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 743                         page_length = PAGE_SIZE - shmem_page_offset;
 744
 745                 /* If we don't overwrite a cacheline completely we need to be
 746                  * careful to have up-to-date data by first clflushing. Don't
 747                  * overcomplicate things and flush the entire patch. */
 748                 partial_cacheline_write = needs_clflush_before &&
 749                         ((shmem_page_offset | page_length)
 750                                 & (boot_cpu_data.x86_clflush_size - 1));
 751
 752                 if (obj->pages) {
 753                         page = obj->pages[offset >> PAGE_SHIFT];
 754                         release_page = 0;
 755                 } else {
 756                         page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
 757                         if (IS_ERR(page)) {
 758                                 ret = PTR_ERR(page);
 759                                 goto out;
 760                         }
 761                         release_page = 1;
 762                 }
 763
 764                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 765                         (page_to_phys(page) & (1 << 17)) != 0;
 766
 767                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 768                                         user_data, page_do_bit17_swizzling,
 769                                         partial_cacheline_write,
 770                                         needs_clflush_after);
 771                 if (ret == 0)
 772                         goto next_page;
 773
 774                 hit_slowpath = 1;
 775                 page_cache_get(page);
 776                 mutex_unlock(&dev->struct_mutex);
 777
 778                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 779                                         user_data, page_do_bit17_swizzling,
 780                                         partial_cacheline_write,
 781                                         needs_clflush_after);
 782
 783                 mutex_lock(&dev->struct_mutex);
 784                 page_cache_release(page);
 785 next_page:
 786                 set_page_dirty(page);
 787                 mark_page_accessed(page);
 788                 if (release_page)
 789                         page_cache_release(page);
 790
 791                 if (ret) {
 792                         ret = -EFAULT;
 793                         goto out;
 794                 }
 795
 796                 remain -= page_length;
 797                 user_data += page_length;
 798                 offset += page_length;
 799         }
 800
 801 out:
 802         if (hit_slowpath) {
 803                 /* Fixup: Kill any reinstated backing storage pages */
 804                 if (obj->madv == __I915_MADV_PURGED)
 805                         i915_gem_object_truncate(obj);
 806                 /* and flush dirty cachelines in case the object isn't in the cpu write
 807                  * domain anymore. */
 808                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 809                         i915_gem_clflush_object(obj);
 810                         intel_gtt_chipset_flush();
 811                 }
 812         }
 813
 814         if (needs_clflush_after)
 815                 intel_gtt_chipset_flush();
 816
 817         return ret;
 818 }
 819
 820 /**
 821  * Writes data to the object referenced by handle.
 822  *
 823  * On error, the contents of the buffer that were to be modified are undefined.
 824  */
 825 int
 826 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 827                       struct drm_file *file)
 828 {
 829         struct drm_i915_gem_pwrite *args = data;
 830         struct drm_i915_gem_object *obj;
 831         int ret;
 832
 833         if (args->size == 0)
 834                 return 0;
 835
 836         if (!access_ok(VERIFY_READ,
 837                        (char __user *)(uintptr_t)args->data_ptr,
 838                        args->size))
 839                 return -EFAULT;
 840
 841         ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
 842                                            args->size);
 843         if (ret)
 844                 return -EFAULT;
 845
 846         ret = i915_mutex_lock_interruptible(dev);
 847         if (ret)
 848                 return ret;
 849
 850         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 851         if (&obj->base == NULL) {
 852                 ret = -ENOENT;
 853                 goto unlock;
 854         }
 855
 856         /* Bounds check destination. */
 857         if (args->offset > obj->base.size ||
 858             args->size > obj->base.size - args->offset) {
 859                 ret = -EINVAL;
 860                 goto out;
 861         }
 862
 863         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 864
 865         ret = -EFAULT;
 866         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 867          * it would end up going through the fenced access, and we'll get
 868          * different detiling behavior between reading and writing.
 869          * pread/pwrite currently are reading and writing from the CPU
 870          * perspective, requiring manual detiling by the client.
 871          */
 872         if (obj->phys_obj) {
 873                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
 874                 goto out;
 875         }
 876
 877         if (obj->gtt_space &&
 878             obj->cache_level == I915_CACHE_NONE &&
 879             obj->map_and_fenceable &&
 880             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 881                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
 882                 /* Note that the gtt paths might fail with non-page-backed user
 883                  * pointers (e.g. gtt mappings when moving data between
 884                  * textures). Fallback to the shmem path in that case. */
 885         }
 886
 887         if (ret == -EFAULT)
 888                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 889
 890 out:
 891         drm_gem_object_unreference(&obj->base);
 892 unlock:
 893         mutex_unlock(&dev->struct_mutex);
 894         return ret;
 895 }
 896
 897 /**
 898  * Called when user space prepares to use an object with the CPU, either
 899  * through the mmap ioctl's mapping or a GTT mapping.
 900  */
 901 int
 902 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 903                           struct drm_file *file)
 904 {
 905         struct drm_i915_gem_set_domain *args = data;
 906         struct drm_i915_gem_object *obj;
 907         uint32_t read_domains = args->read_domains;
 908         uint32_t write_domain = args->write_domain;
 909         int ret;
 910
 911         if (!(dev->driver->driver_features & DRIVER_GEM))
 912                 return -ENODEV;
 913
 914         /* Only handle setting domains to types used by the CPU. */
 915         if (write_domain & I915_GEM_GPU_DOMAINS)
 916                 return -EINVAL;
 917
 918         if (read_domains & I915_GEM_GPU_DOMAINS)
 919                 return -EINVAL;
 920
 921         /* Having something in the write domain implies it's in the read
 922          * domain, and only that read domain.  Enforce that in the request.
 923          */
 924         if (write_domain != 0 && read_domains != write_domain)
 925                 return -EINVAL;
 926
 927         ret = i915_mutex_lock_interruptible(dev);
 928         if (ret)
 929                 return ret;
 930
 931         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 932         if (&obj->base == NULL) {
 933                 ret = -ENOENT;
 934                 goto unlock;
 935         }
 936
 937         if (read_domains & I915_GEM_DOMAIN_GTT) {
 938                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 939
 940                 /* Silently promote "you're not bound, there was nothing to do"
 941                  * to success, since the client was just asking us to
 942                  * make sure everything was done.
 943                  */
 944                 if (ret == -EINVAL)
 945                         ret = 0;
 946         } else {
 947                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 948         }
 949
 950         drm_gem_object_unreference(&obj->base);
 951 unlock:
 952         mutex_unlock(&dev->struct_mutex);
 953         return ret;
 954 }
 955
 956 /**
 957  * Called when user space has done writes to this buffer
 958  */
 959 int
 960 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 961                          struct drm_file *file)
 962 {
 963         struct drm_i915_gem_sw_finish *args = data;
 964         struct drm_i915_gem_object *obj;
 965         int ret = 0;
 966
 967         if (!(dev->driver->driver_features & DRIVER_GEM))
 968                 return -ENODEV;
 969
 970         ret = i915_mutex_lock_interruptible(dev);
 971         if (ret)
 972                 return ret;
 973
 974         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 975         if (&obj->base == NULL) {
 976                 ret = -ENOENT;
 977                 goto unlock;
 978         }
 979
 980         /* Pinned buffers may be scanout, so flush the cache */
 981         if (obj->pin_count)
 982                 i915_gem_object_flush_cpu_write_domain(obj);
 983
 984         drm_gem_object_unreference(&obj->base);
 985 unlock:
 986         mutex_unlock(&dev->struct_mutex);
 987         return ret;
 988 }
 989
 990 /**
 991  * Maps the contents of an object, returning the address it is mapped
 992  * into.
 993  *
 994  * While the mapping holds a reference on the contents of the object, it doesn't
 995  * imply a ref on the object itself.
 996  */
 997 int
 998 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 999                     struct drm_file *file)
1000 {
1001         struct drm_i915_gem_mmap *args = data;
1002         struct drm_gem_object *obj;
1003         unsigned long addr;
1004
1005         if (!(dev->driver->driver_features & DRIVER_GEM))
1006                 return -ENODEV;
1007
1008         obj = drm_gem_object_lookup(dev, file, args->handle);
1009         if (obj == NULL)
1010                 return -ENOENT;
1011
1012         down_write(&current->mm->mmap_sem);
1013         addr = do_mmap(obj->filp, 0, args->size,
1014                        PROT_READ | PROT_WRITE, MAP_SHARED,
1015                        args->offset);
1016         up_write(&current->mm->mmap_sem);
1017         drm_gem_object_unreference_unlocked(obj);
1018         if (IS_ERR((void *)addr))
1019                 return addr;
1020
1021         args->addr_ptr = (uint64_t) addr;
1022
1023         return 0;
1024 }
1025
1026 /**
1027  * i915_gem_fault - fault a page into the GTT
1028  * vma: VMA in question
1029  * vmf: fault info
1030  *
1031  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1032  * from userspace.  The fault handler takes care of binding the object to
1033  * the GTT (if needed), allocating and programming a fence register (again,
1034  * only if needed based on whether the old reg is still valid or the object
1035  * is tiled) and inserting a new PTE into the faulting process.
1036  *
1037  * Note that the faulting process may involve evicting existing objects
1038  * from the GTT and/or fence registers to make room.  So performance may
1039  * suffer if the GTT working set is large or there are few fence registers
1040  * left.
1041  */
1042 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1043 {
1044         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1045         struct drm_device *dev = obj->base.dev;
1046         drm_i915_private_t *dev_priv = dev->dev_private;
1047         pgoff_t page_offset;
1048         unsigned long pfn;
1049         int ret = 0;
1050         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1051
1052         /* We don't use vmf->pgoff since that has the fake offset */
1053         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1054                 PAGE_SHIFT;
1055
1056         ret = i915_mutex_lock_interruptible(dev);
1057         if (ret)
1058                 goto out;
1059
1060         trace_i915_gem_object_fault(obj, page_offset, true, write);
1061
1062         /* Now bind it into the GTT if needed */
1063         if (!obj->map_and_fenceable) {
1064                 ret = i915_gem_object_unbind(obj);
1065                 if (ret)
1066                         goto unlock;
1067         }
1068         if (!obj->gtt_space) {
1069                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1070                 if (ret)
1071                         goto unlock;
1072
1073                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1074                 if (ret)
1075                         goto unlock;
1076         }
1077
1078         if (!obj->has_global_gtt_mapping)
1079                 i915_gem_gtt_bind_object(obj, obj->cache_level);
1080
1081         if (obj->tiling_mode == I915_TILING_NONE)
1082                 ret = i915_gem_object_put_fence(obj);
1083         else
1084                 ret = i915_gem_object_get_fence(obj, NULL);
1085         if (ret)
1086                 goto unlock;
1087
1088         if (i915_gem_object_is_inactive(obj))
1089                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1090
1091         obj->fault_mappable = true;
1092
1093         pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1094                 page_offset;
1095
1096         /* Finally, remap it using the new GTT offset */
1097         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1098 unlock:
1099         mutex_unlock(&dev->struct_mutex);
1100 out:
1101         switch (ret) {
1102         case -EIO:
1103         case -EAGAIN:
1104                 /* Give the error handler a chance to run and move the
1105                  * objects off the GPU active list. Next time we service the
1106                  * fault, we should be able to transition the page into the
1107                  * GTT without touching the GPU (and so avoid further
1108                  * EIO/EGAIN). If the GPU is wedged, then there is no issue
1109                  * with coherency, just lost writes.
1110                  */
1111                 set_need_resched();
1112         case 0:
1113         case -ERESTARTSYS:
1114         case -EINTR:
1115                 return VM_FAULT_NOPAGE;
1116         case -ENOMEM:
1117                 return VM_FAULT_OOM;
1118         default:
1119                 return VM_FAULT_SIGBUS;
1120         }
1121 }
1122
1123 /**
1124  * i915_gem_release_mmap - remove physical page mappings
1125  * @obj: obj in question
1126  *
1127  * Preserve the reservation of the mmapping with the DRM core code, but
1128  * relinquish ownership of the pages back to the system.
1129  *
1130  * It is vital that we remove the page mapping if we have mapped a tiled
1131  * object through the GTT and then lose the fence register due to
1132  * resource pressure. Similarly if the object has been moved out of the
1133  * aperture, than pages mapped into userspace must be revoked. Removing the
1134  * mapping will then trigger a page fault on the next user access, allowing
1135  * fixup by i915_gem_fault().
1136  */
1137 void
1138 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1139 {
1140         if (!obj->fault_mappable)
1141                 return;
1142
1143         if (obj->base.dev->dev_mapping)
1144                 unmap_mapping_range(obj->base.dev->dev_mapping,
1145                                     (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1146                                     obj->base.size, 1);
1147
1148         obj->fault_mappable = false;
1149 }
1150
1151 static uint32_t
1152 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1153 {
1154         uint32_t gtt_size;
1155
1156         if (INTEL_INFO(dev)->gen >= 4 ||
1157             tiling_mode == I915_TILING_NONE)
1158                 return size;
1159
1160         /* Previous chips need a power-of-two fence region when tiling */
1161         if (INTEL_INFO(dev)->gen == 3)
1162                 gtt_size = 1024*1024;
1163         else
1164                 gtt_size = 512*1024;
1165
1166         while (gtt_size < size)
1167                 gtt_size <<= 1;
1168
1169         return gtt_size;
1170 }
1171
1172 /**
1173  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1174  * @obj: object to check
1175  *
1176  * Return the required GTT alignment for an object, taking into account
1177  * potential fence register mapping.
1178  */
1179 static uint32_t
1180 i915_gem_get_gtt_alignment(struct drm_device *dev,
1181                            uint32_t size,
1182                            int tiling_mode)
1183 {
1184         /*
1185          * Minimum alignment is 4k (GTT page size), but might be greater
1186          * if a fence register is needed for the object.
1187          */
1188         if (INTEL_INFO(dev)->gen >= 4 ||
1189             tiling_mode == I915_TILING_NONE)
1190                 return 4096;
1191
1192         /*
1193          * Previous chips need to be aligned to the size of the smallest
1194          * fence register that can contain the object.
1195          */
1196         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1197 }
1198
1199 /**
1200  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1201  *                                       unfenced object
1202  * @dev: the device
1203  * @size: size of the object
1204  * @tiling_mode: tiling mode of the object
1205  *
1206  * Return the required GTT alignment for an object, only taking into account
1207  * unfenced tiled surface requirements.
1208  */
1209 uint32_t
1210 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1211                                     uint32_t size,
1212                                     int tiling_mode)
1213 {
1214         /*
1215          * Minimum alignment is 4k (GTT page size) for sane hw.
1216          */
1217         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1218             tiling_mode == I915_TILING_NONE)
1219                 return 4096;
1220
1221         /* Previous hardware however needs to be aligned to a power-of-two
1222          * tile height. The simplest method for determining this is to reuse
1223          * the power-of-tile object size.
1224          */
1225         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1226 }
1227
1228 int
1229 i915_gem_mmap_gtt(struct drm_file *file,
1230                   struct drm_device *dev,
1231                   uint32_t handle,
1232                   uint64_t *offset)
1233 {
1234         struct drm_i915_private *dev_priv = dev->dev_private;
1235         struct drm_i915_gem_object *obj;
1236         int ret;
1237
1238         if (!(dev->driver->driver_features & DRIVER_GEM))
1239                 return -ENODEV;
1240
1241         ret = i915_mutex_lock_interruptible(dev);
1242         if (ret)
1243                 return ret;
1244
1245         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1246         if (&obj->base == NULL) {
1247                 ret = -ENOENT;
1248                 goto unlock;
1249         }
1250
1251         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1252                 ret = -E2BIG;
1253                 goto out;
1254         }
1255
1256         if (obj->madv != I915_MADV_WILLNEED) {
1257                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1258                 ret = -EINVAL;
1259                 goto out;
1260         }
1261
1262         if (!obj->base.map_list.map) {
1263                 ret = drm_gem_create_mmap_offset(&obj->base);
1264                 if (ret)
1265                         goto out;
1266         }
1267
1268         *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1269
1270 out:
1271         drm_gem_object_unreference(&obj->base);
1272 unlock:
1273         mutex_unlock(&dev->struct_mutex);
1274         return ret;
1275 }
1276
1277 /**
1278  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1279  * @dev: DRM device
1280  * @data: GTT mapping ioctl data
1281  * @file: GEM object info
1282  *
1283  * Simply returns the fake offset to userspace so it can mmap it.
1284  * The mmap call will end up in drm_gem_mmap(), which will set things
1285  * up so we can get faults in the handler above.
1286  *
1287  * The fault handler will take care of binding the object into the GTT
1288  * (since it may have been evicted to make room for something), allocating
1289  * a fence register, and mapping the appropriate aperture address into
1290  * userspace.
1291  */
1292 int
1293 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1294                         struct drm_file *file)
1295 {
1296         struct drm_i915_gem_mmap_gtt *args = data;
1297
1298         if (!(dev->driver->driver_features & DRIVER_GEM))
1299                 return -ENODEV;
1300
1301         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1302 }
1303
1304
1305 static int
1306 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1307                               gfp_t gfpmask)
1308 {
1309         int page_count, i;
1310         struct address_space *mapping;
1311         struct inode *inode;
1312         struct page *page;
1313
1314         /* Get the list of pages out of our struct file.  They'll be pinned
1315          * at this point until we release them.
1316          */
1317         page_count = obj->base.size / PAGE_SIZE;
1318         BUG_ON(obj->pages != NULL);
1319         obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1320         if (obj->pages == NULL)
1321                 return -ENOMEM;
1322
1323         inode = obj->base.filp->f_path.dentry->d_inode;
1324         mapping = inode->i_mapping;
1325         gfpmask |= mapping_gfp_mask(mapping);
1326
1327         for (i = 0; i < page_count; i++) {
1328                 page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
1329                 if (IS_ERR(page))
1330                         goto err_pages;
1331
1332                 obj->pages[i] = page;
1333         }
1334
1335         if (i915_gem_object_needs_bit17_swizzle(obj))
1336                 i915_gem_object_do_bit_17_swizzle(obj);
1337
1338         return 0;
1339
1340 err_pages:
1341         while (i--)
1342                 page_cache_release(obj->pages[i]);
1343
1344         drm_free_large(obj->pages);
1345         obj->pages = NULL;
1346         return PTR_ERR(page);
1347 }
1348
1349 static void
1350 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1351 {
1352         int page_count = obj->base.size / PAGE_SIZE;
1353         int i;
1354
1355         BUG_ON(obj->madv == __I915_MADV_PURGED);
1356
1357         if (i915_gem_object_needs_bit17_swizzle(obj))
1358                 i915_gem_object_save_bit_17_swizzle(obj);
1359
1360         if (obj->madv == I915_MADV_DONTNEED)
1361                 obj->dirty = 0;
1362
1363         for (i = 0; i < page_count; i++) {
1364                 if (obj->dirty)
1365                         set_page_dirty(obj->pages[i]);
1366
1367                 if (obj->madv == I915_MADV_WILLNEED)
1368                         mark_page_accessed(obj->pages[i]);
1369
1370                 page_cache_release(obj->pages[i]);
1371         }
1372         obj->dirty = 0;
1373
1374         drm_free_large(obj->pages);
1375         obj->pages = NULL;
1376 }
1377
1378 void
1379 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1380                                struct intel_ring_buffer *ring,
1381                                u32 seqno)
1382 {
1383         struct drm_device *dev = obj->base.dev;
1384         struct drm_i915_private *dev_priv = dev->dev_private;
1385
1386         BUG_ON(ring == NULL);
1387         obj->ring = ring;
1388
1389         /* Add a reference if we're newly entering the active list. */
1390         if (!obj->active) {
1391                 drm_gem_object_reference(&obj->base);
1392                 obj->active = 1;
1393         }
1394
1395         /* Move from whatever list we were on to the tail of execution. */
1396         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1397         list_move_tail(&obj->ring_list, &ring->active_list);
1398
1399         obj->last_rendering_seqno = seqno;
1400
1401         if (obj->fenced_gpu_access) {
1402                 obj->last_fenced_seqno = seqno;
1403                 obj->last_fenced_ring = ring;
1404
1405                 /* Bump MRU to take account of the delayed flush */
1406                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1407                         struct drm_i915_fence_reg *reg;
1408
1409                         reg = &dev_priv->fence_regs[obj->fence_reg];
1410                         list_move_tail(&reg->lru_list,
1411                                        &dev_priv->mm.fence_list);
1412                 }
1413         }
1414 }
1415
1416 static void
1417 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1418 {
1419         list_del_init(&obj->ring_list);
1420         obj->last_rendering_seqno = 0;
1421 }
1422
1423 static void
1424 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1425 {
1426         struct drm_device *dev = obj->base.dev;
1427         drm_i915_private_t *dev_priv = dev->dev_private;
1428
1429         BUG_ON(!obj->active);
1430         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1431
1432         i915_gem_object_move_off_active(obj);
1433 }
1434
1435 static void
1436 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1437 {
1438         struct drm_device *dev = obj->base.dev;
1439         struct drm_i915_private *dev_priv = dev->dev_private;
1440
1441         if (obj->pin_count != 0)
1442                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1443         else
1444                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1445
1446         BUG_ON(!list_empty(&obj->gpu_write_list));
1447         BUG_ON(!obj->active);
1448         obj->ring = NULL;
1449
1450         i915_gem_object_move_off_active(obj);
1451         obj->fenced_gpu_access = false;
1452
1453         obj->active = 0;
1454         obj->pending_gpu_write = false;
1455         drm_gem_object_unreference(&obj->base);
1456
1457         WARN_ON(i915_verify_lists(dev));
1458 }
1459
1460 /* Immediately discard the backing storage */
1461 static void
1462 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1463 {
1464         struct inode *inode;
1465
1466         /* Our goal here is to return as much of the memory as
1467          * is possible back to the system as we are called from OOM.
1468          * To do this we must instruct the shmfs to drop all of its
1469          * backing pages, *now*.
1470          */
1471         inode = obj->base.filp->f_path.dentry->d_inode;
1472         shmem_truncate_range(inode, 0, (loff_t)-1);
1473
1474         if (obj->base.map_list.map)
1475                 drm_gem_free_mmap_offset(&obj->base);
1476
1477         obj->madv = __I915_MADV_PURGED;
1478 }
1479
1480 static inline int
1481 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1482 {
1483         return obj->madv == I915_MADV_DONTNEED;
1484 }
1485
1486 static void
1487 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1488                                uint32_t flush_domains)
1489 {
1490         struct drm_i915_gem_object *obj, *next;
1491
1492         list_for_each_entry_safe(obj, next,
1493                                  &ring->gpu_write_list,
1494                                  gpu_write_list) {
1495                 if (obj->base.write_domain & flush_domains) {
1496                         uint32_t old_write_domain = obj->base.write_domain;
1497
1498                         obj->base.write_domain = 0;
1499                         list_del_init(&obj->gpu_write_list);
1500                         i915_gem_object_move_to_active(obj, ring,
1501                                                        i915_gem_next_request_seqno(ring));
1502
1503                         trace_i915_gem_object_change_domain(obj,
1504                                                             obj->base.read_domains,
1505                                                             old_write_domain);
1506                 }
1507         }
1508 }
1509
1510 static u32
1511 i915_gem_get_seqno(struct drm_device *dev)
1512 {
1513         drm_i915_private_t *dev_priv = dev->dev_private;
1514         u32 seqno = dev_priv->next_seqno;
1515
1516         /* reserve 0 for non-seqno */
1517         if (++dev_priv->next_seqno == 0)
1518                 dev_priv->next_seqno = 1;
1519
1520         return seqno;
1521 }
1522
1523 u32
1524 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
1525 {
1526         if (ring->outstanding_lazy_request == 0)
1527                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
1528
1529         return ring->outstanding_lazy_request;
1530 }
1531
1532 int
1533 i915_add_request(struct intel_ring_buffer *ring,
1534                  struct drm_file *file,
1535                  struct drm_i915_gem_request *request)
1536 {
1537         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1538         uint32_t seqno;
1539         u32 request_ring_position;
1540         int was_empty;
1541         int ret;
1542
1543         BUG_ON(request == NULL);
1544         seqno = i915_gem_next_request_seqno(ring);
1545
1546         /* Record the position of the start of the request so that
1547          * should we detect the updated seqno part-way through the
1548          * GPU processing the request, we never over-estimate the
1549          * position of the head.
1550          */
1551         request_ring_position = intel_ring_get_tail(ring);
1552
1553         ret = ring->add_request(ring, &seqno);
1554         if (ret)
1555             return ret;
1556
1557         trace_i915_gem_request_add(ring, seqno);
1558
1559         request->seqno = seqno;
1560         request->ring = ring;
1561         request->tail = request_ring_position;
1562         request->emitted_jiffies = jiffies;
1563         was_empty = list_empty(&ring->request_list);
1564         list_add_tail(&request->list, &ring->request_list);
1565
1566         if (file) {
1567                 struct drm_i915_file_private *file_priv = file->driver_priv;
1568
1569                 spin_lock(&file_priv->mm.lock);
1570                 request->file_priv = file_priv;
1571                 list_add_tail(&request->client_list,
1572                               &file_priv->mm.request_list);
1573                 spin_unlock(&file_priv->mm.lock);
1574         }
1575
1576         ring->outstanding_lazy_request = 0;
1577
1578         if (!dev_priv->mm.suspended) {
1579                 if (i915_enable_hangcheck) {
1580                         mod_timer(&dev_priv->hangcheck_timer,
1581                                   jiffies +
1582                                   msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1583                 }
1584                 if (was_empty)
1585                         queue_delayed_work(dev_priv->wq,
1586                                            &dev_priv->mm.retire_work, HZ);
1587         }
1588         return 0;
1589 }
1590
1591 static inline void
1592 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1593 {
1594         struct drm_i915_file_private *file_priv = request->file_priv;
1595
1596         if (!file_priv)
1597                 return;
1598
1599         spin_lock(&file_priv->mm.lock);
1600         if (request->file_priv) {
1601                 list_del(&request->client_list);
1602                 request->file_priv = NULL;
1603         }
1604         spin_unlock(&file_priv->mm.lock);
1605 }
1606
1607 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1608                                       struct intel_ring_buffer *ring)
1609 {
1610         while (!list_empty(&ring->request_list)) {
1611                 struct drm_i915_gem_request *request;
1612
1613                 request = list_first_entry(&ring->request_list,
1614                                            struct drm_i915_gem_request,
1615                                            list);
1616
1617                 list_del(&request->list);
1618                 i915_gem_request_remove_from_client(request);
1619                 kfree(request);
1620         }
1621
1622         while (!list_empty(&ring->active_list)) {
1623                 struct drm_i915_gem_object *obj;
1624
1625                 obj = list_first_entry(&ring->active_list,
1626                                        struct drm_i915_gem_object,
1627                                        ring_list);
1628
1629                 obj->base.write_domain = 0;
1630                 list_del_init(&obj->gpu_write_list);
1631                 i915_gem_object_move_to_inactive(obj);
1632         }
1633 }
1634
1635 static void i915_gem_reset_fences(struct drm_device *dev)
1636 {
1637         struct drm_i915_private *dev_priv = dev->dev_private;
1638         int i;
1639
1640         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1641                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1642                 struct drm_i915_gem_object *obj = reg->obj;
1643
1644                 if (!obj)
1645                         continue;
1646
1647                 if (obj->tiling_mode)
1648                         i915_gem_release_mmap(obj);
1649
1650                 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1651                 reg->obj->fenced_gpu_access = false;
1652                 reg->obj->last_fenced_seqno = 0;
1653                 reg->obj->last_fenced_ring = NULL;
1654                 i915_gem_clear_fence_reg(dev, reg);
1655         }
1656 }
1657
1658 void i915_gem_reset(struct drm_device *dev)
1659 {
1660         struct drm_i915_private *dev_priv = dev->dev_private;
1661         struct drm_i915_gem_object *obj;
1662         int i;
1663
1664         for (i = 0; i < I915_NUM_RINGS; i++)
1665                 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1666
1667         /* Remove anything from the flushing lists. The GPU cache is likely
1668          * to be lost on reset along with the data, so simply move the
1669          * lost bo to the inactive list.
1670          */
1671         while (!list_empty(&dev_priv->mm.flushing_list)) {
1672                 obj = list_first_entry(&dev_priv->mm.flushing_list,
1673                                       struct drm_i915_gem_object,
1674                                       mm_list);
1675
1676                 obj->base.write_domain = 0;
1677                 list_del_init(&obj->gpu_write_list);
1678                 i915_gem_object_move_to_inactive(obj);
1679         }
1680
1681         /* Move everything out of the GPU domains to ensure we do any
1682          * necessary invalidation upon reuse.
1683          */
1684         list_for_each_entry(obj,
1685                             &dev_priv->mm.inactive_list,
1686                             mm_list)
1687         {
1688                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1689         }
1690
1691         /* The fence registers are invalidated so clear them out */
1692         i915_gem_reset_fences(dev);
1693 }
1694
1695 /**
1696  * This function clears the request list as sequence numbers are passed.
1697  */
1698 void
1699 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1700 {
1701         uint32_t seqno;
1702         int i;
1703
1704         if (list_empty(&ring->request_list))
1705                 return;
1706
1707         WARN_ON(i915_verify_lists(ring->dev));
1708
1709         seqno = ring->get_seqno(ring);
1710
1711         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1712                 if (seqno >= ring->sync_seqno[i])
1713                         ring->sync_seqno[i] = 0;
1714
1715         while (!list_empty(&ring->request_list)) {
1716                 struct drm_i915_gem_request *request;
1717
1718                 request = list_first_entry(&ring->request_list,
1719                                            struct drm_i915_gem_request,
1720                                            list);
1721
1722                 if (!i915_seqno_passed(seqno, request->seqno))
1723                         break;
1724
1725                 trace_i915_gem_request_retire(ring, request->seqno);
1726                 /* We know the GPU must have read the request to have
1727                  * sent us the seqno + interrupt, so use the position
1728                  * of tail of the request to update the last known position
1729                  * of the GPU head.
1730                  */
1731                 ring->last_retired_head = request->tail;
1732
1733                 list_del(&request->list);
1734                 i915_gem_request_remove_from_client(request);
1735                 kfree(request);
1736         }
1737
1738         /* Move any buffers on the active list that are no longer referenced
1739          * by the ringbuffer to the flushing/inactive lists as appropriate.
1740          */
1741         while (!list_empty(&ring->active_list)) {
1742                 struct drm_i915_gem_object *obj;
1743
1744                 obj = list_first_entry(&ring->active_list,
1745                                       struct drm_i915_gem_object,
1746                                       ring_list);
1747
1748                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1749                         break;
1750
1751                 if (obj->base.write_domain != 0)
1752                         i915_gem_object_move_to_flushing(obj);
1753                 else
1754                         i915_gem_object_move_to_inactive(obj);
1755         }
1756
1757         if (unlikely(ring->trace_irq_seqno &&
1758                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1759                 ring->irq_put(ring);
1760                 ring->trace_irq_seqno = 0;
1761         }
1762
1763         WARN_ON(i915_verify_lists(ring->dev));
1764 }
1765
1766 void
1767 i915_gem_retire_requests(struct drm_device *dev)
1768 {
1769         drm_i915_private_t *dev_priv = dev->dev_private;
1770         int i;
1771
1772         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1773             struct drm_i915_gem_object *obj, *next;
1774
1775             /* We must be careful that during unbind() we do not
1776              * accidentally infinitely recurse into retire requests.
1777              * Currently:
1778              *   retire -> free -> unbind -> wait -> retire_ring
1779              */
1780             list_for_each_entry_safe(obj, next,
1781                                      &dev_priv->mm.deferred_free_list,
1782                                      mm_list)
1783                     i915_gem_free_object_tail(obj);
1784         }
1785
1786         for (i = 0; i < I915_NUM_RINGS; i++)
1787                 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1788 }
1789
1790 static void
1791 i915_gem_retire_work_handler(struct work_struct *work)
1792 {
1793         drm_i915_private_t *dev_priv;
1794         struct drm_device *dev;
1795         bool idle;
1796         int i;
1797
1798         dev_priv = container_of(work, drm_i915_private_t,
1799                                 mm.retire_work.work);
1800         dev = dev_priv->dev;
1801
1802         /* Come back later if the device is busy... */
1803         if (!mutex_trylock(&dev->struct_mutex)) {
1804                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1805                 return;
1806         }
1807
1808         i915_gem_retire_requests(dev);
1809
1810         /* Send a periodic flush down the ring so we don't hold onto GEM
1811          * objects indefinitely.
1812          */
1813         idle = true;
1814         for (i = 0; i < I915_NUM_RINGS; i++) {
1815                 struct intel_ring_buffer *ring = &dev_priv->ring[i];
1816
1817                 if (!list_empty(&ring->gpu_write_list)) {
1818                         struct drm_i915_gem_request *request;
1819                         int ret;
1820
1821                         ret = i915_gem_flush_ring(ring,
1822                                                   0, I915_GEM_GPU_DOMAINS);
1823                         request = kzalloc(sizeof(*request), GFP_KERNEL);
1824                         if (ret || request == NULL ||
1825                             i915_add_request(ring, NULL, request))
1826                             kfree(request);
1827                 }
1828
1829                 idle &= list_empty(&ring->request_list);
1830         }
1831
1832         if (!dev_priv->mm.suspended && !idle)
1833                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1834
1835         mutex_unlock(&dev->struct_mutex);
1836 }
1837
1838 /**
1839  * Waits for a sequence number to be signaled, and cleans up the
1840  * request and object lists appropriately for that event.
1841  */
1842 int
1843 i915_wait_request(struct intel_ring_buffer *ring,
1844                   uint32_t seqno,
1845                   bool do_retire)
1846 {
1847         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1848         u32 ier;
1849         int ret = 0;
1850
1851         BUG_ON(seqno == 0);
1852
1853         if (atomic_read(&dev_priv->mm.wedged)) {
1854                 struct completion *x = &dev_priv->error_completion;
1855                 bool recovery_complete;
1856                 unsigned long flags;
1857
1858                 /* Give the error handler a chance to run. */
1859                 spin_lock_irqsave(&x->wait.lock, flags);
1860                 recovery_complete = x->done > 0;
1861                 spin_unlock_irqrestore(&x->wait.lock, flags);
1862
1863                 return recovery_complete ? -EIO : -EAGAIN;
1864         }
1865
1866         if (seqno == ring->outstanding_lazy_request) {
1867                 struct drm_i915_gem_request *request;
1868
1869                 request = kzalloc(sizeof(*request), GFP_KERNEL);
1870                 if (request == NULL)
1871                         return -ENOMEM;
1872
1873                 ret = i915_add_request(ring, NULL, request);
1874                 if (ret) {
1875                         kfree(request);
1876                         return ret;
1877                 }
1878
1879                 seqno = request->seqno;
1880         }
1881
1882         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
1883                 if (HAS_PCH_SPLIT(ring->dev))
1884                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1885                 else if (IS_VALLEYVIEW(ring->dev))
1886                         ier = I915_READ(GTIER) | I915_READ(VLV_IER);
1887                 else
1888                         ier = I915_READ(IER);
1889                 if (!ier) {
1890                         DRM_ERROR("something (likely vbetool) disabled "
1891                                   "interrupts, re-enabling\n");
1892                         ring->dev->driver->irq_preinstall(ring->dev);
1893                         ring->dev->driver->irq_postinstall(ring->dev);
1894                 }
1895
1896                 trace_i915_gem_request_wait_begin(ring, seqno);
1897
1898                 ring->waiting_seqno = seqno;
1899                 if (ring->irq_get(ring)) {
1900                         if (dev_priv->mm.interruptible)
1901                                 ret = wait_event_interruptible(ring->irq_queue,
1902                                                                i915_seqno_passed(ring->get_seqno(ring), seqno)
1903                                                                || atomic_read(&dev_priv->mm.wedged));
1904                         else
1905                                 wait_event(ring->irq_queue,
1906                                            i915_seqno_passed(ring->get_seqno(ring), seqno)
1907                                            || atomic_read(&dev_priv->mm.wedged));
1908
1909                         ring->irq_put(ring);
1910                 } else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),
1911                                                              seqno) ||
1912                                            atomic_read(&dev_priv->mm.wedged), 3000))
1913                         ret = -EBUSY;
1914                 ring->waiting_seqno = 0;
1915
1916                 trace_i915_gem_request_wait_end(ring, seqno);
1917         }
1918         if (atomic_read(&dev_priv->mm.wedged))
1919                 ret = -EAGAIN;
1920
1921         /* Directly dispatch request retiring.  While we have the work queue
1922          * to handle this, the waiter on a request often wants an associated
1923          * buffer to have made it to the inactive list, and we would need
1924          * a separate wait queue to handle that.
1925          */
1926         if (ret == 0 && do_retire)
1927                 i915_gem_retire_requests_ring(ring);
1928
1929         return ret;
1930 }
1931
1932 /**
1933  * Ensures that all rendering to the object has completed and the object is
1934  * safe to unbind from the GTT or access from the CPU.
1935  */
1936 int
1937 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
1938 {
1939         int ret;
1940
1941         /* This function only exists to support waiting for existing rendering,
1942          * not for emitting required flushes.
1943          */
1944         BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
1945
1946         /* If there is rendering queued on the buffer being evicted, wait for
1947          * it.
1948          */
1949         if (obj->active) {
1950                 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno,
1951                                         true);
1952                 if (ret)
1953                         return ret;
1954         }
1955
1956         return 0;
1957 }
1958
1959 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
1960 {
1961         u32 old_write_domain, old_read_domains;
1962
1963         /* Act a barrier for all accesses through the GTT */
1964         mb();
1965
1966         /* Force a pagefault for domain tracking on next user access */
1967         i915_gem_release_mmap(obj);
1968
1969         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
1970                 return;
1971
1972         old_read_domains = obj->base.read_domains;
1973         old_write_domain = obj->base.write_domain;
1974
1975         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
1976         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
1977
1978         trace_i915_gem_object_change_domain(obj,
1979                                             old_read_domains,
1980                                             old_write_domain);
1981 }
1982
1983 /**
1984  * Unbinds an object from the GTT aperture.
1985  */
1986 int
1987 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
1988 {
1989         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
1990         int ret = 0;
1991
1992         if (obj->gtt_space == NULL)
1993                 return 0;
1994
1995         if (obj->pin_count != 0) {
1996                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1997                 return -EINVAL;
1998         }
1999
2000         ret = i915_gem_object_finish_gpu(obj);
2001         if (ret == -ERESTARTSYS)
2002                 return ret;
2003         /* Continue on if we fail due to EIO, the GPU is hung so we
2004          * should be safe and we need to cleanup or else we might
2005          * cause memory corruption through use-after-free.
2006          */
2007
2008         i915_gem_object_finish_gtt(obj);
2009
2010         /* Move the object to the CPU domain to ensure that
2011          * any possible CPU writes while it's not in the GTT
2012          * are flushed when we go to remap it.
2013          */
2014         if (ret == 0)
2015                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2016         if (ret == -ERESTARTSYS)
2017                 return ret;
2018         if (ret) {
2019                 /* In the event of a disaster, abandon all caches and
2020                  * hope for the best.
2021                  */
2022                 i915_gem_clflush_object(obj);
2023                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2024         }
2025
2026         /* release the fence reg _after_ flushing */
2027         ret = i915_gem_object_put_fence(obj);
2028         if (ret == -ERESTARTSYS)
2029                 return ret;
2030
2031         trace_i915_gem_object_unbind(obj);
2032
2033         if (obj->has_global_gtt_mapping)
2034                 i915_gem_gtt_unbind_object(obj);
2035         if (obj->has_aliasing_ppgtt_mapping) {
2036                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2037                 obj->has_aliasing_ppgtt_mapping = 0;
2038         }
2039         i915_gem_gtt_finish_object(obj);
2040
2041         i915_gem_object_put_pages_gtt(obj);
2042
2043         list_del_init(&obj->gtt_list);
2044         list_del_init(&obj->mm_list);
2045         /* Avoid an unnecessary call to unbind on rebind. */
2046         obj->map_and_fenceable = true;
2047
2048         drm_mm_put_block(obj->gtt_space);
2049         obj->gtt_space = NULL;
2050         obj->gtt_offset = 0;
2051
2052         if (i915_gem_object_is_purgeable(obj))
2053                 i915_gem_object_truncate(obj);
2054
2055         return ret;
2056 }
2057
2058 int
2059 i915_gem_flush_ring(struct intel_ring_buffer *ring,
2060                     uint32_t invalidate_domains,
2061                     uint32_t flush_domains)
2062 {
2063         int ret;
2064
2065         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2066                 return 0;
2067
2068         trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2069
2070         ret = ring->flush(ring, invalidate_domains, flush_domains);
2071         if (ret)
2072                 return ret;
2073
2074         if (flush_domains & I915_GEM_GPU_DOMAINS)
2075                 i915_gem_process_flushing_list(ring, flush_domains);
2076
2077         return 0;
2078 }
2079
2080 static int i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire)
2081 {
2082         int ret;
2083
2084         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2085                 return 0;
2086
2087         if (!list_empty(&ring->gpu_write_list)) {
2088                 ret = i915_gem_flush_ring(ring,
2089                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2090                 if (ret)
2091                         return ret;
2092         }
2093
2094         return i915_wait_request(ring, i915_gem_next_request_seqno(ring),
2095                                  do_retire);
2096 }
2097
2098 int i915_gpu_idle(struct drm_device *dev, bool do_retire)
2099 {
2100         drm_i915_private_t *dev_priv = dev->dev_private;
2101         int ret, i;
2102
2103         /* Flush everything onto the inactive list. */
2104         for (i = 0; i < I915_NUM_RINGS; i++) {
2105                 ret = i915_ring_idle(&dev_priv->ring[i], do_retire);
2106                 if (ret)
2107                         return ret;
2108         }
2109
2110         return 0;
2111 }
2112
2113 static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2114                                        struct intel_ring_buffer *pipelined)
2115 {
2116         struct drm_device *dev = obj->base.dev;
2117         drm_i915_private_t *dev_priv = dev->dev_private;
2118         u32 size = obj->gtt_space->size;
2119         int regnum = obj->fence_reg;
2120         uint64_t val;
2121
2122         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2123                          0xfffff000) << 32;
2124         val |= obj->gtt_offset & 0xfffff000;
2125         val |= (uint64_t)((obj->stride / 128) - 1) <<
2126                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2127
2128         if (obj->tiling_mode == I915_TILING_Y)
2129                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2130         val |= I965_FENCE_REG_VALID;
2131
2132         if (pipelined) {
2133                 int ret = intel_ring_begin(pipelined, 6);
2134                 if (ret)
2135                         return ret;
2136
2137                 intel_ring_emit(pipelined, MI_NOOP);
2138                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2139                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2140                 intel_ring_emit(pipelined, (u32)val);
2141                 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2142                 intel_ring_emit(pipelined, (u32)(val >> 32));
2143                 intel_ring_advance(pipelined);
2144         } else
2145                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2146
2147         return 0;
2148 }
2149
2150 static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2151                                 struct intel_ring_buffer *pipelined)
2152 {
2153         struct drm_device *dev = obj->base.dev;
2154         drm_i915_private_t *dev_priv = dev->dev_private;
2155         u32 size = obj->gtt_space->size;
2156         int regnum = obj->fence_reg;
2157         uint64_t val;
2158
2159         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2160                     0xfffff000) << 32;
2161         val |= obj->gtt_offset & 0xfffff000;
2162         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2163         if (obj->tiling_mode == I915_TILING_Y)
2164                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2165         val |= I965_FENCE_REG_VALID;
2166
2167         if (pipelined) {
2168                 int ret = intel_ring_begin(pipelined, 6);
2169                 if (ret)
2170                         return ret;
2171
2172                 intel_ring_emit(pipelined, MI_NOOP);
2173                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2174                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2175                 intel_ring_emit(pipelined, (u32)val);
2176                 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2177                 intel_ring_emit(pipelined, (u32)(val >> 32));
2178                 intel_ring_advance(pipelined);
2179         } else
2180                 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2181
2182         return 0;
2183 }
2184
2185 static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2186                                 struct intel_ring_buffer *pipelined)
2187 {
2188         struct drm_device *dev = obj->base.dev;
2189         drm_i915_private_t *dev_priv = dev->dev_private;
2190         u32 size = obj->gtt_space->size;
2191         u32 fence_reg, val, pitch_val;
2192         int tile_width;
2193
2194         if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2195                  (size & -size) != size ||
2196                  (obj->gtt_offset & (size - 1)),
2197                  "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2198                  obj->gtt_offset, obj->map_and_fenceable, size))
2199                 return -EINVAL;
2200
2201         if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2202                 tile_width = 128;
2203         else
2204                 tile_width = 512;
2205
2206         /* Note: pitch better be a power of two tile widths */
2207         pitch_val = obj->stride / tile_width;
2208         pitch_val = ffs(pitch_val) - 1;
2209
2210         val = obj->gtt_offset;
2211         if (obj->tiling_mode == I915_TILING_Y)
2212                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2213         val |= I915_FENCE_SIZE_BITS(size);
2214         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2215         val |= I830_FENCE_REG_VALID;
2216
2217         fence_reg = obj->fence_reg;
2218         if (fence_reg < 8)
2219                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2220         else
2221                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2222
2223         if (pipelined) {
2224                 int ret = intel_ring_begin(pipelined, 4);
2225                 if (ret)
2226                         return ret;
2227
2228                 intel_ring_emit(pipelined, MI_NOOP);
2229                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2230                 intel_ring_emit(pipelined, fence_reg);
2231                 intel_ring_emit(pipelined, val);
2232                 intel_ring_advance(pipelined);
2233         } else
2234                 I915_WRITE(fence_reg, val);
2235
2236         return 0;
2237 }
2238
2239 static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2240                                 struct intel_ring_buffer *pipelined)
2241 {
2242         struct drm_device *dev = obj->base.dev;
2243         drm_i915_private_t *dev_priv = dev->dev_private;
2244         u32 size = obj->gtt_space->size;
2245         int regnum = obj->fence_reg;
2246         uint32_t val;
2247         uint32_t pitch_val;
2248
2249         if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2250                  (size & -size) != size ||
2251                  (obj->gtt_offset & (size - 1)),
2252                  "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2253                  obj->gtt_offset, size))
2254                 return -EINVAL;
2255
2256         pitch_val = obj->stride / 128;
2257         pitch_val = ffs(pitch_val) - 1;
2258
2259         val = obj->gtt_offset;
2260         if (obj->tiling_mode == I915_TILING_Y)
2261                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2262         val |= I830_FENCE_SIZE_BITS(size);
2263         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2264         val |= I830_FENCE_REG_VALID;
2265
2266         if (pipelined) {
2267                 int ret = intel_ring_begin(pipelined, 4);
2268                 if (ret)
2269                         return ret;
2270
2271                 intel_ring_emit(pipelined, MI_NOOP);
2272                 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2273                 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2274                 intel_ring_emit(pipelined, val);
2275                 intel_ring_advance(pipelined);
2276         } else
2277                 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2278
2279         return 0;
2280 }
2281
2282 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2283 {
2284         return i915_seqno_passed(ring->get_seqno(ring), seqno);
2285 }
2286
2287 static int
2288 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2289                             struct intel_ring_buffer *pipelined)
2290 {
2291         int ret;
2292
2293         if (obj->fenced_gpu_access) {
2294                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2295                         ret = i915_gem_flush_ring(obj->last_fenced_ring,
2296                                                   0, obj->base.write_domain);
2297                         if (ret)
2298                                 return ret;
2299                 }
2300
2301                 obj->fenced_gpu_access = false;
2302         }
2303
2304         if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2305                 if (!ring_passed_seqno(obj->last_fenced_ring,
2306                                        obj->last_fenced_seqno)) {
2307                         ret = i915_wait_request(obj->last_fenced_ring,
2308                                                 obj->last_fenced_seqno,
2309                                                 true);
2310                         if (ret)
2311                                 return ret;
2312                 }
2313
2314                 obj->last_fenced_seqno = 0;
2315                 obj->last_fenced_ring = NULL;
2316         }
2317
2318         /* Ensure that all CPU reads are completed before installing a fence
2319          * and all writes before removing the fence.
2320          */
2321         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2322                 mb();
2323
2324         return 0;
2325 }
2326
2327 int
2328 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2329 {
2330         int ret;
2331
2332         if (obj->tiling_mode)
2333                 i915_gem_release_mmap(obj);
2334
2335         ret = i915_gem_object_flush_fence(obj, NULL);
2336         if (ret)
2337                 return ret;
2338
2339         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2340                 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2341
2342                 WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count);
2343                 i915_gem_clear_fence_reg(obj->base.dev,
2344                                          &dev_priv->fence_regs[obj->fence_reg]);
2345
2346                 obj->fence_reg = I915_FENCE_REG_NONE;
2347         }
2348
2349         return 0;
2350 }
2351
2352 static struct drm_i915_fence_reg *
2353 i915_find_fence_reg(struct drm_device *dev,
2354                     struct intel_ring_buffer *pipelined)
2355 {
2356         struct drm_i915_private *dev_priv = dev->dev_private;
2357         struct drm_i915_fence_reg *reg, *first, *avail;
2358         int i;
2359
2360         /* First try to find a free reg */
2361         avail = NULL;
2362         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2363                 reg = &dev_priv->fence_regs[i];
2364                 if (!reg->obj)
2365                         return reg;
2366
2367                 if (!reg->pin_count)
2368                         avail = reg;
2369         }
2370
2371         if (avail == NULL)
2372                 return NULL;
2373
2374         /* None available, try to steal one or wait for a user to finish */
2375         avail = first = NULL;
2376         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2377                 if (reg->pin_count)
2378                         continue;
2379
2380                 if (first == NULL)
2381                         first = reg;
2382
2383                 if (!pipelined ||
2384                     !reg->obj->last_fenced_ring ||
2385                     reg->obj->last_fenced_ring == pipelined) {
2386                         avail = reg;
2387                         break;
2388                 }
2389         }
2390
2391         if (avail == NULL)
2392                 avail = first;
2393
2394         return avail;
2395 }
2396
2397 /**
2398  * i915_gem_object_get_fence - set up a fence reg for an object
2399  * @obj: object to map through a fence reg
2400  * @pipelined: ring on which to queue the change, or NULL for CPU access
2401  * @interruptible: must we wait uninterruptibly for the register to retire?
2402  *
2403  * When mapping objects through the GTT, userspace wants to be able to write
2404  * to them without having to worry about swizzling if the object is tiled.
2405  *
2406  * This function walks the fence regs looking for a free one for @obj,
2407  * stealing one if it can't find any.
2408  *
2409  * It then sets up the reg based on the object's properties: address, pitch
2410  * and tiling format.
2411  */
2412 int
2413 i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2414                           struct intel_ring_buffer *pipelined)
2415 {
2416         struct drm_device *dev = obj->base.dev;
2417         struct drm_i915_private *dev_priv = dev->dev_private;
2418         struct drm_i915_fence_reg *reg;
2419         int ret;
2420
2421         /* XXX disable pipelining. There are bugs. Shocking. */
2422         pipelined = NULL;
2423
2424         /* Just update our place in the LRU if our fence is getting reused. */
2425         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2426                 reg = &dev_priv->fence_regs[obj->fence_reg];
2427                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2428
2429                 if (obj->tiling_changed) {
2430                         ret = i915_gem_object_flush_fence(obj, pipelined);
2431                         if (ret)
2432                                 return ret;
2433
2434                         if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2435                                 pipelined = NULL;
2436
2437                         if (pipelined) {
2438                                 reg->setup_seqno =
2439                                         i915_gem_next_request_seqno(pipelined);
2440                                 obj->last_fenced_seqno = reg->setup_seqno;
2441                                 obj->last_fenced_ring = pipelined;
2442                         }
2443
2444                         goto update;
2445                 }
2446
2447                 if (!pipelined) {
2448                         if (reg->setup_seqno) {
2449                                 if (!ring_passed_seqno(obj->last_fenced_ring,
2450                                                        reg->setup_seqno)) {
2451                                         ret = i915_wait_request(obj->last_fenced_ring,
2452                                                                 reg->setup_seqno,
2453                                                                 true);
2454                                         if (ret)
2455                                                 return ret;
2456                                 }
2457
2458                                 reg->setup_seqno = 0;
2459                         }
2460                 } else if (obj->last_fenced_ring &&
2461                            obj->last_fenced_ring != pipelined) {
2462                         ret = i915_gem_object_flush_fence(obj, pipelined);
2463                         if (ret)
2464                                 return ret;
2465                 }
2466
2467                 return 0;
2468         }
2469
2470         reg = i915_find_fence_reg(dev, pipelined);
2471         if (reg == NULL)
2472                 return -EDEADLK;
2473
2474         ret = i915_gem_object_flush_fence(obj, pipelined);
2475         if (ret)
2476                 return ret;
2477
2478         if (reg->obj) {
2479                 struct drm_i915_gem_object *old = reg->obj;
2480
2481                 drm_gem_object_reference(&old->base);
2482
2483                 if (old->tiling_mode)
2484                         i915_gem_release_mmap(old);
2485
2486                 ret = i915_gem_object_flush_fence(old, pipelined);
2487                 if (ret) {
2488                         drm_gem_object_unreference(&old->base);
2489                         return ret;
2490                 }
2491
2492                 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2493                         pipelined = NULL;
2494
2495                 old->fence_reg = I915_FENCE_REG_NONE;
2496                 old->last_fenced_ring = pipelined;
2497                 old->last_fenced_seqno =
2498                         pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2499
2500                 drm_gem_object_unreference(&old->base);
2501         } else if (obj->last_fenced_seqno == 0)
2502                 pipelined = NULL;
2503
2504         reg->obj = obj;
2505         list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2506         obj->fence_reg = reg - dev_priv->fence_regs;
2507         obj->last_fenced_ring = pipelined;
2508
2509         reg->setup_seqno =
2510                 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2511         obj->last_fenced_seqno = reg->setup_seqno;
2512
2513 update:
2514         obj->tiling_changed = false;
2515         switch (INTEL_INFO(dev)->gen) {
2516         case 7:
2517         case 6:
2518                 ret = sandybridge_write_fence_reg(obj, pipelined);
2519                 break;
2520         case 5:
2521         case 4:
2522                 ret = i965_write_fence_reg(obj, pipelined);
2523                 break;
2524         case 3:
2525                 ret = i915_write_fence_reg(obj, pipelined);
2526                 break;
2527         case 2:
2528                 ret = i830_write_fence_reg(obj, pipelined);
2529                 break;
2530         }
2531
2532         return ret;
2533 }
2534
2535 /**
2536  * i915_gem_clear_fence_reg - clear out fence register info
2537  * @obj: object to clear
2538  *
2539  * Zeroes out the fence register itself and clears out the associated
2540  * data structures in dev_priv and obj.
2541  */
2542 static void
2543 i915_gem_clear_fence_reg(struct drm_device *dev,
2544                          struct drm_i915_fence_reg *reg)
2545 {
2546         drm_i915_private_t *dev_priv = dev->dev_private;
2547         uint32_t fence_reg = reg - dev_priv->fence_regs;
2548
2549         switch (INTEL_INFO(dev)->gen) {
2550         case 7:
2551         case 6:
2552                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
2553                 break;
2554         case 5:
2555         case 4:
2556                 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
2557                 break;
2558         case 3:
2559                 if (fence_reg >= 8)
2560                         fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2561                 else
2562         case 2:
2563                         fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2564
2565                 I915_WRITE(fence_reg, 0);
2566                 break;
2567         }
2568
2569         list_del_init(&reg->lru_list);
2570         reg->obj = NULL;
2571         reg->setup_seqno = 0;
2572         reg->pin_count = 0;
2573 }
2574
2575 /**
2576  * Finds free space in the GTT aperture and binds the object there.
2577  */
2578 static int
2579 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2580                             unsigned alignment,
2581                             bool map_and_fenceable)
2582 {
2583         struct drm_device *dev = obj->base.dev;
2584         drm_i915_private_t *dev_priv = dev->dev_private;
2585         struct drm_mm_node *free_space;
2586         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2587         u32 size, fence_size, fence_alignment, unfenced_alignment;
2588         bool mappable, fenceable;
2589         int ret;
2590
2591         if (obj->madv != I915_MADV_WILLNEED) {
2592                 DRM_ERROR("Attempting to bind a purgeable object\n");
2593                 return -EINVAL;
2594         }
2595
2596         fence_size = i915_gem_get_gtt_size(dev,
2597                                            obj->base.size,
2598                                            obj->tiling_mode);
2599         fence_alignment = i915_gem_get_gtt_alignment(dev,
2600                                                      obj->base.size,
2601                                                      obj->tiling_mode);
2602         unfenced_alignment =
2603                 i915_gem_get_unfenced_gtt_alignment(dev,
2604                                                     obj->base.size,
2605                                                     obj->tiling_mode);
2606
2607         if (alignment == 0)
2608                 alignment = map_and_fenceable ? fence_alignment :
2609                                                 unfenced_alignment;
2610         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2611                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2612                 return -EINVAL;
2613         }
2614
2615         size = map_and_fenceable ? fence_size : obj->base.size;
2616
2617         /* If the object is bigger than the entire aperture, reject it early
2618          * before evicting everything in a vain attempt to find space.
2619          */
2620         if (obj->base.size >
2621             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2622                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2623                 return -E2BIG;
2624         }
2625
2626  search_free:
2627         if (map_and_fenceable)
2628                 free_space =
2629                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2630                                                     size, alignment, 0,
2631                                                     dev_priv->mm.gtt_mappable_end,
2632                                                     0);
2633         else
2634                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2635                                                 size, alignment, 0);
2636
2637         if (free_space != NULL) {
2638                 if (map_and_fenceable)
2639                         obj->gtt_space =
2640                                 drm_mm_get_block_range_generic(free_space,
2641                                                                size, alignment, 0,
2642                                                                dev_priv->mm.gtt_mappable_end,
2643                                                                0);
2644                 else
2645                         obj->gtt_space =
2646                                 drm_mm_get_block(free_space, size, alignment);
2647         }
2648         if (obj->gtt_space == NULL) {
2649                 /* If the gtt is empty and we're still having trouble
2650                  * fitting our object in, we're out of memory.
2651                  */
2652                 ret = i915_gem_evict_something(dev, size, alignment,
2653                                                map_and_fenceable);
2654                 if (ret)
2655                         return ret;
2656
2657                 goto search_free;
2658         }
2659
2660         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2661         if (ret) {
2662                 drm_mm_put_block(obj->gtt_space);
2663                 obj->gtt_space = NULL;
2664
2665                 if (ret == -ENOMEM) {
2666                         /* first try to reclaim some memory by clearing the GTT */
2667                         ret = i915_gem_evict_everything(dev, false);
2668                         if (ret) {
2669                                 /* now try to shrink everyone else */
2670                                 if (gfpmask) {
2671                                         gfpmask = 0;
2672                                         goto search_free;
2673                                 }
2674
2675                                 return -ENOMEM;
2676                         }
2677
2678                         goto search_free;
2679                 }
2680
2681                 return ret;
2682         }
2683
2684         ret = i915_gem_gtt_prepare_object(obj);
2685         if (ret) {
2686                 i915_gem_object_put_pages_gtt(obj);
2687                 drm_mm_put_block(obj->gtt_space);
2688                 obj->gtt_space = NULL;
2689
2690                 if (i915_gem_evict_everything(dev, false))
2691                         return ret;
2692
2693                 goto search_free;
2694         }
2695
2696         if (!dev_priv->mm.aliasing_ppgtt)
2697                 i915_gem_gtt_bind_object(obj, obj->cache_level);
2698
2699         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2700         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2701
2702         /* Assert that the object is not currently in any GPU domain. As it
2703          * wasn't in the GTT, there shouldn't be any way it could have been in
2704          * a GPU cache
2705          */
2706         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2707         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2708
2709         obj->gtt_offset = obj->gtt_space->start;
2710
2711         fenceable =
2712                 obj->gtt_space->size == fence_size &&
2713                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
2714
2715         mappable =
2716                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2717
2718         obj->map_and_fenceable = mappable && fenceable;
2719
2720         trace_i915_gem_object_bind(obj, map_and_fenceable);
2721         return 0;
2722 }
2723
2724 void
2725 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2726 {
2727         /* If we don't have a page list set up, then we're not pinned
2728          * to GPU, and we can ignore the cache flush because it'll happen
2729          * again at bind time.
2730          */
2731         if (obj->pages == NULL)
2732                 return;
2733
2734         /* If the GPU is snooping the contents of the CPU cache,
2735          * we do not need to manually clear the CPU cache lines.  However,
2736          * the caches are only snooped when the render cache is
2737          * flushed/invalidated.  As we always have to emit invalidations
2738          * and flushes when moving into and out of the RENDER domain, correct
2739          * snooping behaviour occurs naturally as the result of our domain
2740          * tracking.
2741          */
2742         if (obj->cache_level != I915_CACHE_NONE)
2743                 return;
2744
2745         trace_i915_gem_object_clflush(obj);
2746
2747         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2748 }
2749
2750 /** Flushes any GPU write domain for the object if it's dirty. */
2751 static int
2752 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2753 {
2754         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2755                 return 0;
2756
2757         /* Queue the GPU write cache flushing we need. */
2758         return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2759 }
2760
2761 /** Flushes the GTT write domain for the object if it's dirty. */
2762 static void
2763 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2764 {
2765         uint32_t old_write_domain;
2766
2767         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2768                 return;
2769
2770         /* No actual flushing is required for the GTT write domain.  Writes
2771          * to it immediately go to main memory as far as we know, so there's
2772          * no chipset flush.  It also doesn't land in render cache.
2773          *
2774          * However, we do have to enforce the order so that all writes through
2775          * the GTT land before any writes to the device, such as updates to
2776          * the GATT itself.
2777          */
2778         wmb();
2779
2780         old_write_domain = obj->base.write_domain;
2781         obj->base.write_domain = 0;
2782
2783         trace_i915_gem_object_change_domain(obj,
2784                                             obj->base.read_domains,
2785                                             old_write_domain);
2786 }
2787
2788 /** Flushes the CPU write domain for the object if it's dirty. */
2789 static void
2790 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2791 {
2792         uint32_t old_write_domain;
2793
2794         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2795                 return;
2796
2797         i915_gem_clflush_object(obj);
2798         intel_gtt_chipset_flush();
2799         old_write_domain = obj->base.write_domain;
2800         obj->base.write_domain = 0;
2801
2802         trace_i915_gem_object_change_domain(obj,
2803                                             obj->base.read_domains,
2804                                             old_write_domain);
2805 }
2806
2807 /**
2808  * Moves a single object to the GTT read, and possibly write domain.
2809  *
2810  * This function returns when the move is complete, including waiting on
2811  * flushes to occur.
2812  */
2813 int
2814 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2815 {
2816         uint32_t old_write_domain, old_read_domains;
2817         int ret;
2818
2819         /* Not valid to be called on unbound objects. */
2820         if (obj->gtt_space == NULL)
2821                 return -EINVAL;
2822
2823         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2824                 return 0;
2825
2826         ret = i915_gem_object_flush_gpu_write_domain(obj);
2827         if (ret)
2828                 return ret;
2829
2830         if (obj->pending_gpu_write || write) {
2831                 ret = i915_gem_object_wait_rendering(obj);
2832                 if (ret)
2833                         return ret;
2834         }
2835
2836         i915_gem_object_flush_cpu_write_domain(obj);
2837
2838         old_write_domain = obj->base.write_domain;
2839         old_read_domains = obj->base.read_domains;
2840
2841         /* It should now be out of any other write domains, and we can update
2842          * the domain values for our changes.
2843          */
2844         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2845         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2846         if (write) {
2847                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2848                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2849                 obj->dirty = 1;
2850         }
2851
2852         trace_i915_gem_object_change_domain(obj,
2853                                             old_read_domains,
2854                                             old_write_domain);
2855
2856         return 0;
2857 }
2858
2859 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2860                                     enum i915_cache_level cache_level)
2861 {
2862         struct drm_device *dev = obj->base.dev;
2863         drm_i915_private_t *dev_priv = dev->dev_private;
2864         int ret;
2865
2866         if (obj->cache_level == cache_level)
2867                 return 0;
2868
2869         if (obj->pin_count) {
2870                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2871                 return -EBUSY;
2872         }
2873
2874         if (obj->gtt_space) {
2875                 ret = i915_gem_object_finish_gpu(obj);
2876                 if (ret)
2877                         return ret;
2878
2879                 i915_gem_object_finish_gtt(obj);
2880
2881                 /* Before SandyBridge, you could not use tiling or fence
2882                  * registers with snooped memory, so relinquish any fences
2883                  * currently pointing to our region in the aperture.
2884                  */
2885                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2886                         ret = i915_gem_object_put_fence(obj);
2887                         if (ret)
2888                                 return ret;
2889                 }
2890
2891                 if (obj->has_global_gtt_mapping)
2892                         i915_gem_gtt_bind_object(obj, cache_level);
2893                 if (obj->has_aliasing_ppgtt_mapping)
2894                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2895                                                obj, cache_level);
2896         }
2897
2898         if (cache_level == I915_CACHE_NONE) {
2899                 u32 old_read_domains, old_write_domain;
2900
2901                 /* If we're coming from LLC cached, then we haven't
2902                  * actually been tracking whether the data is in the
2903                  * CPU cache or not, since we only allow one bit set
2904                  * in obj->write_domain and have been skipping the clflushes.
2905                  * Just set it to the CPU cache for now.
2906                  */
2907                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
2908                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
2909
2910                 old_read_domains = obj->base.read_domains;
2911                 old_write_domain = obj->base.write_domain;
2912
2913                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
2914                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2915
2916                 trace_i915_gem_object_change_domain(obj,
2917                                                     old_read_domains,
2918                                                     old_write_domain);
2919         }
2920
2921         obj->cache_level = cache_level;
2922         return 0;
2923 }
2924
2925 /*
2926  * Prepare buffer for display plane (scanout, cursors, etc).
2927  * Can be called from an uninterruptible phase (modesetting) and allows
2928  * any flushes to be pipelined (for pageflips).
2929  *
2930  * For the display plane, we want to be in the GTT but out of any write
2931  * domains. So in many ways this looks like set_to_gtt_domain() apart from the
2932  * ability to pipeline the waits, pinning and any additional subtleties
2933  * that may differentiate the display plane from ordinary buffers.
2934  */
2935 int
2936 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2937                                      u32 alignment,
2938                                      struct intel_ring_buffer *pipelined)
2939 {
2940         u32 old_read_domains, old_write_domain;
2941         int ret;
2942
2943         ret = i915_gem_object_flush_gpu_write_domain(obj);
2944         if (ret)
2945                 return ret;
2946
2947         if (pipelined != obj->ring) {
2948                 ret = i915_gem_object_wait_rendering(obj);
2949                 if (ret == -ERESTARTSYS)
2950                         return ret;
2951         }
2952
2953         /* The display engine is not coherent with the LLC cache on gen6.  As
2954          * a result, we make sure that the pinning that is about to occur is
2955          * done with uncached PTEs. This is lowest common denominator for all
2956          * chipsets.
2957          *
2958          * However for gen6+, we could do better by using the GFDT bit instead
2959          * of uncaching, which would allow us to flush all the LLC-cached data
2960          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
2961          */
2962         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2963         if (ret)
2964                 return ret;
2965
2966         /* As the user may map the buffer once pinned in the display plane
2967          * (e.g. libkms for the bootup splash), we have to ensure that we
2968          * always use map_and_fenceable for all scanout buffers.
2969          */
2970         ret = i915_gem_object_pin(obj, alignment, true);
2971         if (ret)
2972                 return ret;
2973
2974         i915_gem_object_flush_cpu_write_domain(obj);
2975
2976         old_write_domain = obj->base.write_domain;
2977         old_read_domains = obj->base.read_domains;
2978
2979         /* It should now be out of any other write domains, and we can update
2980          * the domain values for our changes.
2981          */
2982         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2983         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2984
2985         trace_i915_gem_object_change_domain(obj,
2986                                             old_read_domains,
2987                                             old_write_domain);
2988
2989         return 0;
2990 }
2991
2992 int
2993 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2994 {
2995         int ret;
2996
2997         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2998                 return 0;
2999
3000         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3001                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
3002                 if (ret)
3003                         return ret;
3004         }
3005
3006         ret = i915_gem_object_wait_rendering(obj);
3007         if (ret)
3008                 return ret;
3009
3010         /* Ensure that we invalidate the GPU's caches and TLBs. */
3011         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3012         return 0;
3013 }
3014
3015 /**
3016  * Moves a single object to the CPU read, and possibly write domain.
3017  *
3018  * This function returns when the move is complete, including waiting on
3019  * flushes to occur.
3020  */
3021 int
3022 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3023 {
3024         uint32_t old_write_domain, old_read_domains;
3025         int ret;
3026
3027         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3028                 return 0;
3029
3030         ret = i915_gem_object_flush_gpu_write_domain(obj);
3031         if (ret)
3032                 return ret;
3033
3034         ret = i915_gem_object_wait_rendering(obj);
3035         if (ret)
3036                 return ret;
3037
3038         i915_gem_object_flush_gtt_write_domain(obj);
3039
3040         old_write_domain = obj->base.write_domain;
3041         old_read_domains = obj->base.read_domains;
3042
3043         /* Flush the CPU cache if it's still invalid. */
3044         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3045                 i915_gem_clflush_object(obj);
3046
3047                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3048         }
3049
3050         /* It should now be out of any other write domains, and we can update
3051          * the domain values for our changes.
3052          */
3053         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3054
3055         /* If we're writing through the CPU, then the GPU read domains will
3056          * need to be invalidated at next use.
3057          */
3058         if (write) {
3059                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3060                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3061         }
3062
3063         trace_i915_gem_object_change_domain(obj,
3064                                             old_read_domains,
3065                                             old_write_domain);
3066
3067         return 0;
3068 }
3069
3070 /* Throttle our rendering by waiting until the ring has completed our requests
3071  * emitted over 20 msec ago.
3072  *
3073  * Note that if we were to use the current jiffies each time around the loop,
3074  * we wouldn't escape the function with any frames outstanding if the time to
3075  * render a frame was over 20ms.
3076  *
3077  * This should get us reasonable parallelism between CPU and GPU but also
3078  * relatively low latency when blocking on a particular request to finish.
3079  */
3080 static int
3081 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3082 {
3083         struct drm_i915_private *dev_priv = dev->dev_private;
3084         struct drm_i915_file_private *file_priv = file->driver_priv;
3085         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3086         struct drm_i915_gem_request *request;
3087         struct intel_ring_buffer *ring = NULL;
3088         u32 seqno = 0;
3089         int ret;
3090
3091         if (atomic_read(&dev_priv->mm.wedged))
3092                 return -EIO;
3093
3094         spin_lock(&file_priv->mm.lock);
3095         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3096                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3097                         break;
3098
3099                 ring = request->ring;
3100                 seqno = request->seqno;
3101         }
3102         spin_unlock(&file_priv->mm.lock);
3103
3104         if (seqno == 0)
3105                 return 0;
3106
3107         ret = 0;
3108         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3109                 /* And wait for the seqno passing without holding any locks and
3110                  * causing extra latency for others. This is safe as the irq
3111                  * generation is designed to be run atomically and so is
3112                  * lockless.
3113                  */
3114                 if (ring->irq_get(ring)) {
3115                         ret = wait_event_interruptible(ring->irq_queue,
3116                                                        i915_seqno_passed(ring->get_seqno(ring), seqno)
3117                                                        || atomic_read(&dev_priv->mm.wedged));
3118                         ring->irq_put(ring);
3119
3120                         if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3121                                 ret = -EIO;
3122                 } else if (wait_for_atomic(i915_seqno_passed(ring->get_seqno(ring),
3123                                                              seqno) ||
3124                                     atomic_read(&dev_priv->mm.wedged), 3000)) {
3125                         ret = -EBUSY;
3126                 }
3127         }
3128
3129         if (ret == 0)
3130                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3131
3132         return ret;
3133 }
3134
3135 int
3136 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3137                     uint32_t alignment,
3138                     bool map_and_fenceable)
3139 {
3140         struct drm_device *dev = obj->base.dev;
3141         struct drm_i915_private *dev_priv = dev->dev_private;
3142         int ret;
3143
3144         BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
3145         WARN_ON(i915_verify_lists(dev));
3146
3147         if (obj->gtt_space != NULL) {
3148                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3149                     (map_and_fenceable && !obj->map_and_fenceable)) {
3150                         WARN(obj->pin_count,
3151                              "bo is already pinned with incorrect alignment:"
3152                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3153                              " obj->map_and_fenceable=%d\n",
3154                              obj->gtt_offset, alignment,
3155                              map_and_fenceable,
3156                              obj->map_and_fenceable);
3157                         ret = i915_gem_object_unbind(obj);
3158                         if (ret)
3159                                 return ret;
3160                 }
3161         }
3162
3163         if (obj->gtt_space == NULL) {
3164                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3165                                                   map_and_fenceable);
3166                 if (ret)
3167                         return ret;
3168         }
3169
3170         if (!obj->has_global_gtt_mapping && map_and_fenceable)
3171                 i915_gem_gtt_bind_object(obj, obj->cache_level);
3172
3173         if (obj->pin_count++ == 0) {
3174                 if (!obj->active)
3175                         list_move_tail(&obj->mm_list,
3176                                        &dev_priv->mm.pinned_list);
3177         }
3178         obj->pin_mappable |= map_and_fenceable;
3179
3180         WARN_ON(i915_verify_lists(dev));
3181         return 0;
3182 }
3183
3184 void
3185 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3186 {
3187         struct drm_device *dev = obj->base.dev;
3188         drm_i915_private_t *dev_priv = dev->dev_private;
3189
3190         WARN_ON(i915_verify_lists(dev));
3191         BUG_ON(obj->pin_count == 0);
3192         BUG_ON(obj->gtt_space == NULL);
3193
3194         if (--obj->pin_count == 0) {
3195                 if (!obj->active)
3196                         list_move_tail(&obj->mm_list,
3197                                        &dev_priv->mm.inactive_list);
3198                 obj->pin_mappable = false;
3199         }
3200         WARN_ON(i915_verify_lists(dev));
3201 }
3202
3203 int
3204 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3205                    struct drm_file *file)
3206 {
3207         struct drm_i915_gem_pin *args = data;
3208         struct drm_i915_gem_object *obj;
3209         int ret;
3210
3211         ret = i915_mutex_lock_interruptible(dev);
3212         if (ret)
3213                 return ret;
3214
3215         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3216         if (&obj->base == NULL) {
3217                 ret = -ENOENT;
3218                 goto unlock;
3219         }
3220
3221         if (obj->madv != I915_MADV_WILLNEED) {
3222                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3223                 ret = -EINVAL;
3224                 goto out;
3225         }
3226
3227         if (obj->pin_filp != NULL && obj->pin_filp != file) {
3228                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3229                           args->handle);
3230                 ret = -EINVAL;
3231                 goto out;
3232         }
3233
3234         obj->user_pin_count++;
3235         obj->pin_filp = file;
3236         if (obj->user_pin_count == 1) {
3237                 ret = i915_gem_object_pin(obj, args->alignment, true);
3238                 if (ret)
3239                         goto out;
3240         }
3241
3242         /* XXX - flush the CPU caches for pinned objects
3243          * as the X server doesn't manage domains yet
3244          */
3245         i915_gem_object_flush_cpu_write_domain(obj);
3246         args->offset = obj->gtt_offset;
3247 out:
3248         drm_gem_object_unreference(&obj->base);
3249 unlock:
3250         mutex_unlock(&dev->struct_mutex);
3251         return ret;
3252 }
3253
3254 int
3255 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3256                      struct drm_file *file)
3257 {
3258         struct drm_i915_gem_pin *args = data;
3259         struct drm_i915_gem_object *obj;
3260         int ret;
3261
3262         ret = i915_mutex_lock_interruptible(dev);
3263         if (ret)
3264                 return ret;
3265
3266         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3267         if (&obj->base == NULL) {
3268                 ret = -ENOENT;
3269                 goto unlock;
3270         }
3271
3272         if (obj->pin_filp != file) {
3273                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3274                           args->handle);
3275                 ret = -EINVAL;
3276                 goto out;
3277         }
3278         obj->user_pin_count--;
3279         if (obj->user_pin_count == 0) {
3280                 obj->pin_filp = NULL;
3281                 i915_gem_object_unpin(obj);
3282         }
3283
3284 out:
3285         drm_gem_object_unreference(&obj->base);
3286 unlock:
3287         mutex_unlock(&dev->struct_mutex);
3288         return ret;
3289 }
3290
3291 int
3292 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3293                     struct drm_file *file)
3294 {
3295         struct drm_i915_gem_busy *args = data;
3296         struct drm_i915_gem_object *obj;
3297         int ret;
3298
3299         ret = i915_mutex_lock_interruptible(dev);
3300         if (ret)
3301                 return ret;
3302
3303         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3304         if (&obj->base == NULL) {
3305                 ret = -ENOENT;
3306                 goto unlock;
3307         }
3308
3309         /* Count all active objects as busy, even if they are currently not used
3310          * by the gpu. Users of this interface expect objects to eventually
3311          * become non-busy without any further actions, therefore emit any
3312          * necessary flushes here.
3313          */
3314         args->busy = obj->active;
3315         if (args->busy) {
3316                 /* Unconditionally flush objects, even when the gpu still uses this
3317                  * object. Userspace calling this function indicates that it wants to
3318                  * use this buffer rather sooner than later, so issuing the required
3319                  * flush earlier is beneficial.
3320                  */
3321                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3322                         ret = i915_gem_flush_ring(obj->ring,
3323                                                   0, obj->base.write_domain);
3324                 } else if (obj->ring->outstanding_lazy_request ==
3325                            obj->last_rendering_seqno) {
3326                         struct drm_i915_gem_request *request;
3327
3328                         /* This ring is not being cleared by active usage,
3329                          * so emit a request to do so.
3330                          */
3331                         request = kzalloc(sizeof(*request), GFP_KERNEL);
3332                         if (request) {
3333                                 ret = i915_add_request(obj->ring, NULL, request);
3334                                 if (ret)
3335                                         kfree(request);
3336                         } else
3337                                 ret = -ENOMEM;
3338                 }
3339
3340                 /* Update the active list for the hardware's current position.
3341                  * Otherwise this only updates on a delayed timer or when irqs
3342                  * are actually unmasked, and our working set ends up being
3343                  * larger than required.
3344                  */
3345                 i915_gem_retire_requests_ring(obj->ring);
3346
3347                 args->busy = obj->active;
3348         }
3349
3350         drm_gem_object_unreference(&obj->base);
3351 unlock:
3352         mutex_unlock(&dev->struct_mutex);
3353         return ret;
3354 }
3355
3356 int
3357 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3358                         struct drm_file *file_priv)
3359 {
3360         return i915_gem_ring_throttle(dev, file_priv);
3361 }
3362
3363 int
3364 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3365                        struct drm_file *file_priv)
3366 {
3367         struct drm_i915_gem_madvise *args = data;
3368         struct drm_i915_gem_object *obj;
3369         int ret;
3370
3371         switch (args->madv) {
3372         case I915_MADV_DONTNEED:
3373         case I915_MADV_WILLNEED:
3374             break;
3375         default:
3376             return -EINVAL;
3377         }
3378
3379         ret = i915_mutex_lock_interruptible(dev);
3380         if (ret)
3381                 return ret;
3382
3383         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3384         if (&obj->base == NULL) {
3385                 ret = -ENOENT;
3386                 goto unlock;
3387         }
3388
3389         if (obj->pin_count) {
3390                 ret = -EINVAL;
3391                 goto out;
3392         }
3393
3394         if (obj->madv != __I915_MADV_PURGED)
3395                 obj->madv = args->madv;
3396
3397         /* if the object is no longer bound, discard its backing storage */
3398         if (i915_gem_object_is_purgeable(obj) &&
3399             obj->gtt_space == NULL)
3400                 i915_gem_object_truncate(obj);
3401
3402         args->retained = obj->madv != __I915_MADV_PURGED;
3403
3404 out:
3405         drm_gem_object_unreference(&obj->base);
3406 unlock:
3407         mutex_unlock(&dev->struct_mutex);
3408         return ret;
3409 }
3410
3411 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3412                                                   size_t size)
3413 {
3414         struct drm_i915_private *dev_priv = dev->dev_private;
3415         struct drm_i915_gem_object *obj;
3416         struct address_space *mapping;
3417
3418         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3419         if (obj == NULL)
3420                 return NULL;
3421
3422         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3423                 kfree(obj);
3424                 return NULL;
3425         }
3426
3427         mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3428         mapping_set_gfp_mask(mapping, GFP_HIGHUSER | __GFP_RECLAIMABLE);
3429
3430         i915_gem_info_add_obj(dev_priv, size);
3431
3432         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3433         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3434
3435         if (HAS_LLC(dev)) {
3436                 /* On some devices, we can have the GPU use the LLC (the CPU
3437                  * cache) for about a 10% performance improvement
3438                  * compared to uncached.  Graphics requests other than
3439                  * display scanout are coherent with the CPU in
3440                  * accessing this cache.  This means in this mode we
3441                  * don't need to clflush on the CPU side, and on the
3442                  * GPU side we only need to flush internal caches to
3443                  * get data visible to the CPU.
3444                  *
3445                  * However, we maintain the display planes as UC, and so
3446                  * need to rebind when first used as such.
3447                  */
3448                 obj->cache_level = I915_CACHE_LLC;
3449         } else
3450                 obj->cache_level = I915_CACHE_NONE;
3451
3452         obj->base.driver_private = NULL;
3453         obj->fence_reg = I915_FENCE_REG_NONE;
3454         INIT_LIST_HEAD(&obj->mm_list);
3455         INIT_LIST_HEAD(&obj->gtt_list);
3456         INIT_LIST_HEAD(&obj->ring_list);
3457         INIT_LIST_HEAD(&obj->exec_list);
3458         INIT_LIST_HEAD(&obj->gpu_write_list);
3459         obj->madv = I915_MADV_WILLNEED;
3460         /* Avoid an unnecessary call to unbind on the first bind. */
3461         obj->map_and_fenceable = true;
3462
3463         return obj;
3464 }
3465
3466 int i915_gem_init_object(struct drm_gem_object *obj)
3467 {
3468         BUG();
3469
3470         return 0;
3471 }
3472
3473 static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
3474 {
3475         struct drm_device *dev = obj->base.dev;
3476         drm_i915_private_t *dev_priv = dev->dev_private;
3477         int ret;
3478
3479         ret = i915_gem_object_unbind(obj);
3480         if (ret == -ERESTARTSYS) {
3481                 list_move(&obj->mm_list,
3482                           &dev_priv->mm.deferred_free_list);
3483                 return;
3484         }
3485
3486         trace_i915_gem_object_destroy(obj);
3487
3488         if (obj->base.map_list.map)
3489                 drm_gem_free_mmap_offset(&obj->base);
3490
3491         drm_gem_object_release(&obj->base);
3492         i915_gem_info_remove_obj(dev_priv, obj->base.size);
3493
3494         kfree(obj->bit_17);
3495         kfree(obj);
3496 }
3497
3498 void i915_gem_free_object(struct drm_gem_object *gem_obj)
3499 {
3500         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3501         struct drm_device *dev = obj->base.dev;
3502
3503         while (obj->pin_count > 0)
3504                 i915_gem_object_unpin(obj);
3505
3506         if (obj->phys_obj)
3507                 i915_gem_detach_phys_object(dev, obj);
3508
3509         i915_gem_free_object_tail(obj);
3510 }
3511
3512 int
3513 i915_gem_idle(struct drm_device *dev)
3514 {
3515         drm_i915_private_t *dev_priv = dev->dev_private;
3516         int ret;
3517
3518         mutex_lock(&dev->struct_mutex);
3519
3520         if (dev_priv->mm.suspended) {
3521                 mutex_unlock(&dev->struct_mutex);
3522                 return 0;
3523         }
3524
3525         ret = i915_gpu_idle(dev, true);
3526         if (ret) {
3527                 mutex_unlock(&dev->struct_mutex);
3528                 return ret;
3529         }
3530
3531         /* Under UMS, be paranoid and evict. */
3532         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
3533                 ret = i915_gem_evict_inactive(dev, false);
3534                 if (ret) {
3535                         mutex_unlock(&dev->struct_mutex);
3536                         return ret;
3537                 }
3538         }
3539
3540         i915_gem_reset_fences(dev);
3541
3542         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
3543          * We need to replace this with a semaphore, or something.
3544          * And not confound mm.suspended!
3545          */
3546         dev_priv->mm.suspended = 1;
3547         del_timer_sync(&dev_priv->hangcheck_timer);
3548
3549         i915_kernel_lost_context(dev);
3550         i915_gem_cleanup_ringbuffer(dev);
3551
3552         mutex_unlock(&dev->struct_mutex);
3553
3554         /* Cancel the retire work handler, which should be idle now. */
3555         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3556
3557         return 0;
3558 }
3559
3560 void i915_gem_init_swizzling(struct drm_device *dev)
3561 {
3562         drm_i915_private_t *dev_priv = dev->dev_private;
3563
3564         if (INTEL_INFO(dev)->gen < 5 ||
3565             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
3566                 return;
3567
3568         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
3569                                  DISP_TILE_SURFACE_SWIZZLING);
3570
3571         if (IS_GEN5(dev))
3572                 return;
3573
3574         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
3575         if (IS_GEN6(dev))
3576                 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB));
3577         else
3578                 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB));
3579 }
3580
3581 void i915_gem_init_ppgtt(struct drm_device *dev)
3582 {
3583         drm_i915_private_t *dev_priv = dev->dev_private;
3584         uint32_t pd_offset;
3585         struct intel_ring_buffer *ring;
3586         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
3587         uint32_t __iomem *pd_addr;
3588         uint32_t pd_entry;
3589         int i;
3590
3591         if (!dev_priv->mm.aliasing_ppgtt)
3592                 return;
3593
3594
3595         pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
3596         for (i = 0; i < ppgtt->num_pd_entries; i++) {
3597                 dma_addr_t pt_addr;
3598
3599                 if (dev_priv->mm.gtt->needs_dmar)
3600                         pt_addr = ppgtt->pt_dma_addr[i];
3601                 else
3602                         pt_addr = page_to_phys(ppgtt->pt_pages[i]);
3603
3604                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
3605                 pd_entry |= GEN6_PDE_VALID;
3606
3607                 writel(pd_entry, pd_addr + i);
3608         }
3609         readl(pd_addr);
3610
3611         pd_offset = ppgtt->pd_offset;
3612         pd_offset /= 64; /* in cachelines, */
3613         pd_offset <<= 16;
3614
3615         if (INTEL_INFO(dev)->gen == 6) {
3616                 uint32_t ecochk = I915_READ(GAM_ECOCHK);
3617                 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
3618                                        ECOCHK_PPGTT_CACHE64B);
3619                 I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE));
3620         } else if (INTEL_INFO(dev)->gen >= 7) {
3621                 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
3622                 /* GFX_MODE is per-ring on gen7+ */
3623         }
3624
3625         for (i = 0; i < I915_NUM_RINGS; i++) {
3626                 ring = &dev_priv->ring[i];
3627
3628                 if (INTEL_INFO(dev)->gen >= 7)
3629                         I915_WRITE(RING_MODE_GEN7(ring),
3630                                    GFX_MODE_ENABLE(GFX_PPGTT_ENABLE));
3631
3632                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
3633                 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
3634         }
3635 }
3636
3637 int
3638 i915_gem_init_hw(struct drm_device *dev)
3639 {
3640         drm_i915_private_t *dev_priv = dev->dev_private;
3641         int ret;
3642
3643         i915_gem_init_swizzling(dev);
3644
3645         ret = intel_init_render_ring_buffer(dev);
3646         if (ret)
3647                 return ret;
3648
3649         if (HAS_BSD(dev)) {
3650                 ret = intel_init_bsd_ring_buffer(dev);
3651                 if (ret)
3652                         goto cleanup_render_ring;
3653         }
3654
3655         if (HAS_BLT(dev)) {
3656                 ret = intel_init_blt_ring_buffer(dev);
3657                 if (ret)
3658                         goto cleanup_bsd_ring;
3659         }
3660
3661         dev_priv->next_seqno = 1;
3662
3663         i915_gem_init_ppgtt(dev);
3664
3665         return 0;
3666
3667 cleanup_bsd_ring:
3668         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
3669 cleanup_render_ring:
3670         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3671         return ret;
3672 }
3673
3674 void
3675 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3676 {
3677         drm_i915_private_t *dev_priv = dev->dev_private;
3678         int i;
3679
3680         for (i = 0; i < I915_NUM_RINGS; i++)
3681                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
3682 }
3683
3684 int
3685 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3686                        struct drm_file *file_priv)
3687 {
3688         drm_i915_private_t *dev_priv = dev->dev_private;
3689         int ret, i;
3690
3691         if (drm_core_check_feature(dev, DRIVER_MODESET))
3692                 return 0;
3693
3694         if (atomic_read(&dev_priv->mm.wedged)) {
3695                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
3696                 atomic_set(&dev_priv->mm.wedged, 0);
3697         }
3698
3699         mutex_lock(&dev->struct_mutex);
3700         dev_priv->mm.suspended = 0;
3701
3702         ret = i915_gem_init_hw(dev);
3703         if (ret != 0) {
3704                 mutex_unlock(&dev->struct_mutex);
3705                 return ret;
3706         }
3707
3708         BUG_ON(!list_empty(&dev_priv->mm.active_list));
3709         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3710         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
3711         for (i = 0; i < I915_NUM_RINGS; i++) {
3712                 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
3713                 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3714         }
3715         mutex_unlock(&dev->struct_mutex);
3716
3717         ret = drm_irq_install(dev);
3718         if (ret)
3719                 goto cleanup_ringbuffer;
3720
3721         return 0;
3722
3723 cleanup_ringbuffer:
3724         mutex_lock(&dev->struct_mutex);
3725         i915_gem_cleanup_ringbuffer(dev);
3726         dev_priv->mm.suspended = 1;
3727         mutex_unlock(&dev->struct_mutex);
3728
3729         return ret;
3730 }
3731
3732 int
3733 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3734                        struct drm_file *file_priv)
3735 {
3736         if (drm_core_check_feature(dev, DRIVER_MODESET))
3737                 return 0;
3738
3739         drm_irq_uninstall(dev);
3740         return i915_gem_idle(dev);
3741 }
3742
3743 void
3744 i915_gem_lastclose(struct drm_device *dev)
3745 {
3746         int ret;
3747
3748         if (drm_core_check_feature(dev, DRIVER_MODESET))
3749                 return;
3750
3751         ret = i915_gem_idle(dev);
3752         if (ret)
3753                 DRM_ERROR("failed to idle hardware: %d\n", ret);
3754 }
3755
3756 static void
3757 init_ring_lists(struct intel_ring_buffer *ring)
3758 {
3759         INIT_LIST_HEAD(&ring->active_list);
3760         INIT_LIST_HEAD(&ring->request_list);
3761         INIT_LIST_HEAD(&ring->gpu_write_list);
3762 }
3763
3764 void
3765 i915_gem_load(struct drm_device *dev)
3766 {
3767         int i;
3768         drm_i915_private_t *dev_priv = dev->dev_private;
3769
3770         INIT_LIST_HEAD(&dev_priv->mm.active_list);
3771         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3772         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3773         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
3774         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3775         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
3776         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3777         for (i = 0; i < I915_NUM_RINGS; i++)
3778                 init_ring_lists(&dev_priv->ring[i]);
3779         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3780                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3781         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3782                           i915_gem_retire_work_handler);
3783         init_completion(&dev_priv->error_completion);
3784
3785         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3786         if (IS_GEN3(dev)) {
3787                 u32 tmp = I915_READ(MI_ARB_STATE);
3788                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
3789                         /* arb state is a masked write, so set bit + bit in mask */
3790                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
3791                         I915_WRITE(MI_ARB_STATE, tmp);
3792                 }
3793         }
3794
3795         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3796
3797         /* Old X drivers will take 0-2 for front, back, depth buffers */
3798         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3799                 dev_priv->fence_reg_start = 3;
3800
3801         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3802                 dev_priv->num_fence_regs = 16;
3803         else
3804                 dev_priv->num_fence_regs = 8;
3805
3806         /* Initialize fence registers to zero */
3807         for (i = 0; i < dev_priv->num_fence_regs; i++) {
3808                 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
3809         }
3810
3811         i915_gem_detect_bit_6_swizzle(dev);
3812         init_waitqueue_head(&dev_priv->pending_flip_queue);
3813
3814         dev_priv->mm.interruptible = true;
3815
3816         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3817         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3818         register_shrinker(&dev_priv->mm.inactive_shrinker);
3819 }
3820
3821 /*
3822  * Create a physically contiguous memory object for this object
3823  * e.g. for cursor + overlay regs
3824  */
3825 static int i915_gem_init_phys_object(struct drm_device *dev,
3826                                      int id, int size, int align)
3827 {
3828         drm_i915_private_t *dev_priv = dev->dev_private;
3829         struct drm_i915_gem_phys_object *phys_obj;
3830         int ret;
3831
3832         if (dev_priv->mm.phys_objs[id - 1] || !size)
3833                 return 0;
3834
3835         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
3836         if (!phys_obj)
3837                 return -ENOMEM;
3838
3839         phys_obj->id = id;
3840
3841         phys_obj->handle = drm_pci_alloc(dev, size, align);
3842         if (!phys_obj->handle) {
3843                 ret = -ENOMEM;
3844                 goto kfree_obj;
3845         }
3846 #ifdef CONFIG_X86
3847         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3848 #endif
3849
3850         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3851
3852         return 0;
3853 kfree_obj:
3854         kfree(phys_obj);
3855         return ret;
3856 }
3857
3858 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
3859 {
3860         drm_i915_private_t *dev_priv = dev->dev_private;
3861         struct drm_i915_gem_phys_object *phys_obj;
3862
3863         if (!dev_priv->mm.phys_objs[id - 1])
3864                 return;
3865
3866         phys_obj = dev_priv->mm.phys_objs[id - 1];
3867         if (phys_obj->cur_obj) {
3868                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
3869         }
3870
3871 #ifdef CONFIG_X86
3872         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3873 #endif
3874         drm_pci_free(dev, phys_obj->handle);
3875         kfree(phys_obj);
3876         dev_priv->mm.phys_objs[id - 1] = NULL;
3877 }
3878
3879 void i915_gem_free_all_phys_object(struct drm_device *dev)
3880 {
3881         int i;
3882
3883         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
3884                 i915_gem_free_phys_object(dev, i);
3885 }
3886
3887 void i915_gem_detach_phys_object(struct drm_device *dev,
3888                                  struct drm_i915_gem_object *obj)
3889 {
3890         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3891         char *vaddr;
3892         int i;
3893         int page_count;
3894
3895         if (!obj->phys_obj)
3896                 return;
3897         vaddr = obj->phys_obj->handle->vaddr;
3898
3899         page_count = obj->base.size / PAGE_SIZE;
3900         for (i = 0; i < page_count; i++) {
3901                 struct page *page = shmem_read_mapping_page(mapping, i);
3902                 if (!IS_ERR(page)) {
3903                         char *dst = kmap_atomic(page);
3904                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
3905                         kunmap_atomic(dst);
3906
3907                         drm_clflush_pages(&page, 1);
3908
3909                         set_page_dirty(page);
3910                         mark_page_accessed(page);
3911                         page_cache_release(page);
3912                 }
3913         }
3914         intel_gtt_chipset_flush();
3915
3916         obj->phys_obj->cur_obj = NULL;
3917         obj->phys_obj = NULL;
3918 }
3919
3920 int
3921 i915_gem_attach_phys_object(struct drm_device *dev,
3922                             struct drm_i915_gem_object *obj,
3923                             int id,
3924                             int align)
3925 {
3926         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3927         drm_i915_private_t *dev_priv = dev->dev_private;
3928         int ret = 0;
3929         int page_count;
3930         int i;
3931
3932         if (id > I915_MAX_PHYS_OBJECT)
3933                 return -EINVAL;
3934
3935         if (obj->phys_obj) {
3936                 if (obj->phys_obj->id == id)
3937                         return 0;
3938                 i915_gem_detach_phys_object(dev, obj);
3939         }
3940
3941         /* create a new object */
3942         if (!dev_priv->mm.phys_objs[id - 1]) {
3943                 ret = i915_gem_init_phys_object(dev, id,
3944                                                 obj->base.size, align);
3945                 if (ret) {
3946                         DRM_ERROR("failed to init phys object %d size: %zu\n",
3947                                   id, obj->base.size);
3948                         return ret;
3949                 }
3950         }
3951
3952         /* bind to the object */
3953         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3954         obj->phys_obj->cur_obj = obj;
3955
3956         page_count = obj->base.size / PAGE_SIZE;
3957
3958         for (i = 0; i < page_count; i++) {
3959                 struct page *page;
3960                 char *dst, *src;
3961
3962                 page = shmem_read_mapping_page(mapping, i);
3963                 if (IS_ERR(page))
3964                         return PTR_ERR(page);
3965
3966                 src = kmap_atomic(page);
3967                 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
3968                 memcpy(dst, src, PAGE_SIZE);
3969                 kunmap_atomic(src);
3970
3971                 mark_page_accessed(page);
3972                 page_cache_release(page);
3973         }
3974
3975         return 0;
3976 }
3977
3978 static int
3979 i915_gem_phys_pwrite(struct drm_device *dev,
3980                      struct drm_i915_gem_object *obj,
3981                      struct drm_i915_gem_pwrite *args,
3982                      struct drm_file *file_priv)
3983 {
3984         void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
3985         char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
3986
3987         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
3988                 unsigned long unwritten;
3989
3990                 /* The physical object once assigned is fixed for the lifetime
3991                  * of the obj, so we can safely drop the lock and continue
3992                  * to access vaddr.
3993                  */
3994                 mutex_unlock(&dev->struct_mutex);
3995                 unwritten = copy_from_user(vaddr, user_data, args->size);
3996                 mutex_lock(&dev->struct_mutex);
3997                 if (unwritten)
3998                         return -EFAULT;
3999         }
4000
4001         intel_gtt_chipset_flush();
4002         return 0;
4003 }
4004
4005 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4006 {
4007         struct drm_i915_file_private *file_priv = file->driver_priv;
4008
4009         /* Clean up our request list when the client is going away, so that
4010          * later retire_requests won't dereference our soon-to-be-gone
4011          * file_priv.
4012          */
4013         spin_lock(&file_priv->mm.lock);
4014         while (!list_empty(&file_priv->mm.request_list)) {
4015                 struct drm_i915_gem_request *request;
4016
4017                 request = list_first_entry(&file_priv->mm.request_list,
4018                                            struct drm_i915_gem_request,
4019                                            client_list);
4020                 list_del(&request->client_list);
4021                 request->file_priv = NULL;
4022         }
4023         spin_unlock(&file_priv->mm.lock);
4024 }
4025
4026 static int
4027 i915_gpu_is_active(struct drm_device *dev)
4028 {
4029         drm_i915_private_t *dev_priv = dev->dev_private;
4030         int lists_empty;
4031
4032         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4033                       list_empty(&dev_priv->mm.active_list);
4034
4035         return !lists_empty;
4036 }
4037
4038 static int
4039 i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
4040 {
4041         struct drm_i915_private *dev_priv =
4042                 container_of(shrinker,
4043                              struct drm_i915_private,
4044                              mm.inactive_shrinker);
4045         struct drm_device *dev = dev_priv->dev;
4046         struct drm_i915_gem_object *obj, *next;
4047         int nr_to_scan = sc->nr_to_scan;
4048         int cnt;
4049
4050         if (!mutex_trylock(&dev->struct_mutex))
4051                 return 0;
4052
4053         /* "fast-path" to count number of available objects */
4054         if (nr_to_scan == 0) {
4055                 cnt = 0;
4056                 list_for_each_entry(obj,
4057                                     &dev_priv->mm.inactive_list,
4058                                     mm_list)
4059                         cnt++;
4060                 mutex_unlock(&dev->struct_mutex);
4061                 return cnt / 100 * sysctl_vfs_cache_pressure;
4062         }
4063
4064 rescan:
4065         /* first scan for clean buffers */
4066         i915_gem_retire_requests(dev);
4067
4068         list_for_each_entry_safe(obj, next,
4069                                  &dev_priv->mm.inactive_list,
4070                                  mm_list) {
4071                 if (i915_gem_object_is_purgeable(obj)) {
4072                         if (i915_gem_object_unbind(obj) == 0 &&
4073                             --nr_to_scan == 0)
4074                                 break;
4075                 }
4076         }
4077
4078         /* second pass, evict/count anything still on the inactive list */
4079         cnt = 0;
4080         list_for_each_entry_safe(obj, next,
4081                                  &dev_priv->mm.inactive_list,
4082                                  mm_list) {
4083                 if (nr_to_scan &&
4084                     i915_gem_object_unbind(obj) == 0)
4085                         nr_to_scan--;
4086                 else
4087                         cnt++;
4088         }
4089
4090         if (nr_to_scan && i915_gpu_is_active(dev)) {
4091                 /*
4092                  * We are desperate for pages, so as a last resort, wait
4093                  * for the GPU to finish and discard whatever we can.
4094                  * This has a dramatic impact to reduce the number of
4095                  * OOM-killer events whilst running the GPU aggressively.
4096                  */
4097                 if (i915_gpu_idle(dev, true) == 0)
4098                         goto rescan;
4099         }
4100         mutex_unlock(&dev->struct_mutex);
4101         return cnt / 100 * sysctl_vfs_cache_pressure;
4102 }