drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/shmem_fs.h>
  35 #include <linux/slab.h>
  36 #include <linux/swap.h>
  37 #include <linux/pci.h>
  38 #include <linux/dma-buf.h>
  39
  40 static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
  41 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  42 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  43 static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  44                                                     unsigned alignment,
  45                                                     bool map_and_fenceable);
  46 static int i915_gem_phys_pwrite(struct drm_device *dev,
  47                                 struct drm_i915_gem_object *obj,
  48                                 struct drm_i915_gem_pwrite *args,
  49                                 struct drm_file *file);
  50
  51 static void i915_gem_write_fence(struct drm_device *dev, int reg,
  52                                  struct drm_i915_gem_object *obj);
  53 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
  54                                          struct drm_i915_fence_reg *fence,
  55                                          bool enable);
  56
  57 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  58                                     struct shrink_control *sc);
  59 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
  60
  61 static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
  62 {
  63         if (obj->tiling_mode)
  64                 i915_gem_release_mmap(obj);
  65
  66         /* As we do not have an associated fence register, we will force
  67          * a tiling change if we ever need to acquire one.
  68          */
  69         obj->fence_dirty = false;
  70         obj->fence_reg = I915_FENCE_REG_NONE;
  71 }
  72
  73 /* some bookkeeping */
  74 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  75                                   size_t size)
  76 {
  77         dev_priv->mm.object_count++;
  78         dev_priv->mm.object_memory += size;
  79 }
  80
  81 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  82                                      size_t size)
  83 {
  84         dev_priv->mm.object_count--;
  85         dev_priv->mm.object_memory -= size;
  86 }
  87
  88 static int
  89 i915_gem_wait_for_error(struct drm_device *dev)
  90 {
  91         struct drm_i915_private *dev_priv = dev->dev_private;
  92         struct completion *x = &dev_priv->error_completion;
  93         unsigned long flags;
  94         int ret;
  95
  96         if (!atomic_read(&dev_priv->mm.wedged))
  97                 return 0;
  98
  99         /*
 100          * Only wait 10 seconds for the gpu reset to complete to avoid hanging
 101          * userspace. If it takes that long something really bad is going on and
 102          * we should simply try to bail out and fail as gracefully as possible.
 103          */
 104         ret = wait_for_completion_interruptible_timeout(x, 10*HZ);
 105         if (ret == 0) {
 106                 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
 107                 return -EIO;
 108         } else if (ret < 0) {
 109                 return ret;
 110         }
 111
 112         if (atomic_read(&dev_priv->mm.wedged)) {
 113                 /* GPU is hung, bump the completion count to account for
 114                  * the token we just consumed so that we never hit zero and
 115                  * end up waiting upon a subsequent completion event that
 116                  * will never happen.
 117                  */
 118                 spin_lock_irqsave(&x->wait.lock, flags);
 119                 x->done++;
 120                 spin_unlock_irqrestore(&x->wait.lock, flags);
 121         }
 122         return 0;
 123 }
 124
 125 int i915_mutex_lock_interruptible(struct drm_device *dev)
 126 {
 127         int ret;
 128
 129         ret = i915_gem_wait_for_error(dev);
 130         if (ret)
 131                 return ret;
 132
 133         ret = mutex_lock_interruptible(&dev->struct_mutex);
 134         if (ret)
 135                 return ret;
 136
 137         WARN_ON(i915_verify_lists(dev));
 138         return 0;
 139 }
 140
 141 static inline bool
 142 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 143 {
 144         return !obj->active;
 145 }
 146
 147 int
 148 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 149                     struct drm_file *file)
 150 {
 151         struct drm_i915_gem_init *args = data;
 152
 153         if (drm_core_check_feature(dev, DRIVER_MODESET))
 154                 return -ENODEV;
 155
 156         if (args->gtt_start >= args->gtt_end ||
 157             (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
 158                 return -EINVAL;
 159
 160         /* GEM with user mode setting was never supported on ilk and later. */
 161         if (INTEL_INFO(dev)->gen >= 5)
 162                 return -ENODEV;
 163
 164         mutex_lock(&dev->struct_mutex);
 165         i915_gem_init_global_gtt(dev, args->gtt_start,
 166                                  args->gtt_end, args->gtt_end);
 167         mutex_unlock(&dev->struct_mutex);
 168
 169         return 0;
 170 }
 171
 172 int
 173 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 174                             struct drm_file *file)
 175 {
 176         struct drm_i915_private *dev_priv = dev->dev_private;
 177         struct drm_i915_gem_get_aperture *args = data;
 178         struct drm_i915_gem_object *obj;
 179         size_t pinned;
 180
 181         pinned = 0;
 182         mutex_lock(&dev->struct_mutex);
 183         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
 184                 if (obj->pin_count)
 185                         pinned += obj->gtt_space->size;
 186         mutex_unlock(&dev->struct_mutex);
 187
 188         args->aper_size = dev_priv->mm.gtt_total;
 189         args->aper_available_size = args->aper_size - pinned;
 190
 191         return 0;
 192 }
 193
 194 static int
 195 i915_gem_create(struct drm_file *file,
 196                 struct drm_device *dev,
 197                 uint64_t size,
 198                 uint32_t *handle_p)
 199 {
 200         struct drm_i915_gem_object *obj;
 201         int ret;
 202         u32 handle;
 203
 204         size = roundup(size, PAGE_SIZE);
 205         if (size == 0)
 206                 return -EINVAL;
 207
 208         /* Allocate the new object */
 209         obj = i915_gem_alloc_object(dev, size);
 210         if (obj == NULL)
 211                 return -ENOMEM;
 212
 213         ret = drm_gem_handle_create(file, &obj->base, &handle);
 214         if (ret) {
 215                 drm_gem_object_release(&obj->base);
 216                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 217                 kfree(obj);
 218                 return ret;
 219         }
 220
 221         /* drop reference from allocate - handle holds it now */
 222         drm_gem_object_unreference(&obj->base);
 223         trace_i915_gem_object_create(obj);
 224
 225         *handle_p = handle;
 226         return 0;
 227 }
 228
 229 int
 230 i915_gem_dumb_create(struct drm_file *file,
 231                      struct drm_device *dev,
 232                      struct drm_mode_create_dumb *args)
 233 {
 234         /* have to work out size/pitch and return them */
 235         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
 236         args->size = args->pitch * args->height;
 237         return i915_gem_create(file, dev,
 238                                args->size, &args->handle);
 239 }
 240
 241 int i915_gem_dumb_destroy(struct drm_file *file,
 242                           struct drm_device *dev,
 243                           uint32_t handle)
 244 {
 245         return drm_gem_handle_delete(file, handle);
 246 }
 247
 248 /**
 249  * Creates a new mm object and returns a handle to it.
 250  */
 251 int
 252 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 253                       struct drm_file *file)
 254 {
 255         struct drm_i915_gem_create *args = data;
 256
 257         return i915_gem_create(file, dev,
 258                                args->size, &args->handle);
 259 }
 260
 261 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 262 {
 263         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
 264
 265         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 266                 obj->tiling_mode != I915_TILING_NONE;
 267 }
 268
 269 static inline int
 270 __copy_to_user_swizzled(char __user *cpu_vaddr,
 271                         const char *gpu_vaddr, int gpu_offset,
 272                         int length)
 273 {
 274         int ret, cpu_offset = 0;
 275
 276         while (length > 0) {
 277                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 278                 int this_length = min(cacheline_end - gpu_offset, length);
 279                 int swizzled_gpu_offset = gpu_offset ^ 64;
 280
 281                 ret = __copy_to_user(cpu_vaddr + cpu_offset,
 282                                      gpu_vaddr + swizzled_gpu_offset,
 283                                      this_length);
 284                 if (ret)
 285                         return ret + length;
 286
 287                 cpu_offset += this_length;
 288                 gpu_offset += this_length;
 289                 length -= this_length;
 290         }
 291
 292         return 0;
 293 }
 294
 295 static inline int
 296 __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
 297                           const char __user *cpu_vaddr,
 298                           int length)
 299 {
 300         int ret, cpu_offset = 0;
 301
 302         while (length > 0) {
 303                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 304                 int this_length = min(cacheline_end - gpu_offset, length);
 305                 int swizzled_gpu_offset = gpu_offset ^ 64;
 306
 307                 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
 308                                        cpu_vaddr + cpu_offset,
 309                                        this_length);
 310                 if (ret)
 311                         return ret + length;
 312
 313                 cpu_offset += this_length;
 314                 gpu_offset += this_length;
 315                 length -= this_length;
 316         }
 317
 318         return 0;
 319 }
 320
 321 /* Per-page copy function for the shmem pread fastpath.
 322  * Flushes invalid cachelines before reading the target if
 323  * needs_clflush is set. */
 324 static int
 325 shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
 326                  char __user *user_data,
 327                  bool page_do_bit17_swizzling, bool needs_clflush)
 328 {
 329         char *vaddr;
 330         int ret;
 331
 332         if (unlikely(page_do_bit17_swizzling))
 333                 return -EINVAL;
 334
 335         vaddr = kmap_atomic(page);
 336         if (needs_clflush)
 337                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 338                                        page_length);
 339         ret = __copy_to_user_inatomic(user_data,
 340                                       vaddr + shmem_page_offset,
 341                                       page_length);
 342         kunmap_atomic(vaddr);
 343
 344         return ret;
 345 }
 346
 347 static void
 348 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 349                              bool swizzled)
 350 {
 351         if (unlikely(swizzled)) {
 352                 unsigned long start = (unsigned long) addr;
 353                 unsigned long end = (unsigned long) addr + length;
 354
 355                 /* For swizzling simply ensure that we always flush both
 356                  * channels. Lame, but simple and it works. Swizzled
 357                  * pwrite/pread is far from a hotpath - current userspace
 358                  * doesn't use it at all. */
 359                 start = round_down(start, 128);
 360                 end = round_up(end, 128);
 361
 362                 drm_clflush_virt_range((void *)start, end - start);
 363         } else {
 364                 drm_clflush_virt_range(addr, length);
 365         }
 366
 367 }
 368
 369 /* Only difference to the fast-path function is that this can handle bit17
 370  * and uses non-atomic copy and kmap functions. */
 371 static int
 372 shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 373                  char __user *user_data,
 374                  bool page_do_bit17_swizzling, bool needs_clflush)
 375 {
 376         char *vaddr;
 377         int ret;
 378
 379         vaddr = kmap(page);
 380         if (needs_clflush)
 381                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 382                                              page_length,
 383                                              page_do_bit17_swizzling);
 384
 385         if (page_do_bit17_swizzling)
 386                 ret = __copy_to_user_swizzled(user_data,
 387                                               vaddr, shmem_page_offset,
 388                                               page_length);
 389         else
 390                 ret = __copy_to_user(user_data,
 391                                      vaddr + shmem_page_offset,
 392                                      page_length);
 393         kunmap(page);
 394
 395         return ret;
 396 }
 397
 398 static int
 399 i915_gem_shmem_pread(struct drm_device *dev,
 400                      struct drm_i915_gem_object *obj,
 401                      struct drm_i915_gem_pread *args,
 402                      struct drm_file *file)
 403 {
 404         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 405         char __user *user_data;
 406         ssize_t remain;
 407         loff_t offset;
 408         int shmem_page_offset, page_length, ret = 0;
 409         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 410         int hit_slowpath = 0;
 411         int prefaulted = 0;
 412         int needs_clflush = 0;
 413         int release_page;
 414
 415         user_data = (char __user *) (uintptr_t) args->data_ptr;
 416         remain = args->size;
 417
 418         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 419
 420         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
 421                 /* If we're not in the cpu read domain, set ourself into the gtt
 422                  * read domain and manually flush cachelines (if required). This
 423                  * optimizes for the case when the gpu will dirty the data
 424                  * anyway again before the next pread happens. */
 425                 if (obj->cache_level == I915_CACHE_NONE)
 426                         needs_clflush = 1;
 427                 ret = i915_gem_object_set_to_gtt_domain(obj, false);
 428                 if (ret)
 429                         return ret;
 430         }
 431
 432         offset = args->offset;
 433
 434         while (remain > 0) {
 435                 struct page *page;
 436
 437                 /* Operation in this page
 438                  *
 439                  * shmem_page_offset = offset within page in shmem file
 440                  * page_length = bytes to copy for this page
 441                  */
 442                 shmem_page_offset = offset_in_page(offset);
 443                 page_length = remain;
 444                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 445                         page_length = PAGE_SIZE - shmem_page_offset;
 446
 447                 if (obj->pages) {
 448                         page = obj->pages[offset >> PAGE_SHIFT];
 449                         release_page = 0;
 450                 } else {
 451                         page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
 452                         if (IS_ERR(page)) {
 453                                 ret = PTR_ERR(page);
 454                                 goto out;
 455                         }
 456                         release_page = 1;
 457                 }
 458
 459                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 460                         (page_to_phys(page) & (1 << 17)) != 0;
 461
 462                 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
 463                                        user_data, page_do_bit17_swizzling,
 464                                        needs_clflush);
 465                 if (ret == 0)
 466                         goto next_page;
 467
 468                 hit_slowpath = 1;
 469                 page_cache_get(page);
 470                 mutex_unlock(&dev->struct_mutex);
 471
 472                 if (!prefaulted) {
 473                         ret = fault_in_multipages_writeable(user_data, remain);
 474                         /* Userspace is tricking us, but we've already clobbered
 475                          * its pages with the prefault and promised to write the
 476                          * data up to the first fault. Hence ignore any errors
 477                          * and just continue. */
 478                         (void)ret;
 479                         prefaulted = 1;
 480                 }
 481
 482                 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
 483                                        user_data, page_do_bit17_swizzling,
 484                                        needs_clflush);
 485
 486                 mutex_lock(&dev->struct_mutex);
 487                 page_cache_release(page);
 488 next_page:
 489                 mark_page_accessed(page);
 490                 if (release_page)
 491                         page_cache_release(page);
 492
 493                 if (ret) {
 494                         ret = -EFAULT;
 495                         goto out;
 496                 }
 497
 498                 remain -= page_length;
 499                 user_data += page_length;
 500                 offset += page_length;
 501         }
 502
 503 out:
 504         if (hit_slowpath) {
 505                 /* Fixup: Kill any reinstated backing storage pages */
 506                 if (obj->madv == __I915_MADV_PURGED)
 507                         i915_gem_object_truncate(obj);
 508         }
 509
 510         return ret;
 511 }
 512
 513 /**
 514  * Reads data from the object referenced by handle.
 515  *
 516  * On error, the contents of *data are undefined.
 517  */
 518 int
 519 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 520                      struct drm_file *file)
 521 {
 522         struct drm_i915_gem_pread *args = data;
 523         struct drm_i915_gem_object *obj;
 524         int ret = 0;
 525
 526         if (args->size == 0)
 527                 return 0;
 528
 529         if (!access_ok(VERIFY_WRITE,
 530                        (char __user *)(uintptr_t)args->data_ptr,
 531                        args->size))
 532                 return -EFAULT;
 533
 534         ret = i915_mutex_lock_interruptible(dev);
 535         if (ret)
 536                 return ret;
 537
 538         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 539         if (&obj->base == NULL) {
 540                 ret = -ENOENT;
 541                 goto unlock;
 542         }
 543
 544         /* Bounds check source.  */
 545         if (args->offset > obj->base.size ||
 546             args->size > obj->base.size - args->offset) {
 547                 ret = -EINVAL;
 548                 goto out;
 549         }
 550
 551         /* prime objects have no backing filp to GEM pread/pwrite
 552          * pages from.
 553          */
 554         if (!obj->base.filp) {
 555                 ret = -EINVAL;
 556                 goto out;
 557         }
 558
 559         trace_i915_gem_object_pread(obj, args->offset, args->size);
 560
 561         ret = i915_gem_shmem_pread(dev, obj, args, file);
 562
 563 out:
 564         drm_gem_object_unreference(&obj->base);
 565 unlock:
 566         mutex_unlock(&dev->struct_mutex);
 567         return ret;
 568 }
 569
 570 /* This is the fast write path which cannot handle
 571  * page faults in the source data
 572  */
 573
 574 static inline int
 575 fast_user_write(struct io_mapping *mapping,
 576                 loff_t page_base, int page_offset,
 577                 char __user *user_data,
 578                 int length)
 579 {
 580         void __iomem *vaddr_atomic;
 581         void *vaddr;
 582         unsigned long unwritten;
 583
 584         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 585         /* We can use the cpu mem copy function because this is X86. */
 586         vaddr = (void __force*)vaddr_atomic + page_offset;
 587         unwritten = __copy_from_user_inatomic_nocache(vaddr,
 588                                                       user_data, length);
 589         io_mapping_unmap_atomic(vaddr_atomic);
 590         return unwritten;
 591 }
 592
 593 /**
 594  * This is the fast pwrite path, where we copy the data directly from the
 595  * user into the GTT, uncached.
 596  */
 597 static int
 598 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 599                          struct drm_i915_gem_object *obj,
 600                          struct drm_i915_gem_pwrite *args,
 601                          struct drm_file *file)
 602 {
 603         drm_i915_private_t *dev_priv = dev->dev_private;
 604         ssize_t remain;
 605         loff_t offset, page_base;
 606         char __user *user_data;
 607         int page_offset, page_length, ret;
 608
 609         ret = i915_gem_object_pin(obj, 0, true);
 610         if (ret)
 611                 goto out;
 612
 613         ret = i915_gem_object_set_to_gtt_domain(obj, true);
 614         if (ret)
 615                 goto out_unpin;
 616
 617         ret = i915_gem_object_put_fence(obj);
 618         if (ret)
 619                 goto out_unpin;
 620
 621         user_data = (char __user *) (uintptr_t) args->data_ptr;
 622         remain = args->size;
 623
 624         offset = obj->gtt_offset + args->offset;
 625
 626         while (remain > 0) {
 627                 /* Operation in this page
 628                  *
 629                  * page_base = page offset within aperture
 630                  * page_offset = offset within page
 631                  * page_length = bytes to copy for this page
 632                  */
 633                 page_base = offset & PAGE_MASK;
 634                 page_offset = offset_in_page(offset);
 635                 page_length = remain;
 636                 if ((page_offset + remain) > PAGE_SIZE)
 637                         page_length = PAGE_SIZE - page_offset;
 638
 639                 /* If we get a fault while copying data, then (presumably) our
 640                  * source page isn't available.  Return the error and we'll
 641                  * retry in the slow path.
 642                  */
 643                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 644                                     page_offset, user_data, page_length)) {
 645                         ret = -EFAULT;
 646                         goto out_unpin;
 647                 }
 648
 649                 remain -= page_length;
 650                 user_data += page_length;
 651                 offset += page_length;
 652         }
 653
 654 out_unpin:
 655         i915_gem_object_unpin(obj);
 656 out:
 657         return ret;
 658 }
 659
 660 /* Per-page copy function for the shmem pwrite fastpath.
 661  * Flushes invalid cachelines before writing to the target if
 662  * needs_clflush_before is set and flushes out any written cachelines after
 663  * writing if needs_clflush is set. */
 664 static int
 665 shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
 666                   char __user *user_data,
 667                   bool page_do_bit17_swizzling,
 668                   bool needs_clflush_before,
 669                   bool needs_clflush_after)
 670 {
 671         char *vaddr;
 672         int ret;
 673
 674         if (unlikely(page_do_bit17_swizzling))
 675                 return -EINVAL;
 676
 677         vaddr = kmap_atomic(page);
 678         if (needs_clflush_before)
 679                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 680                                        page_length);
 681         ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
 682                                                 user_data,
 683                                                 page_length);
 684         if (needs_clflush_after)
 685                 drm_clflush_virt_range(vaddr + shmem_page_offset,
 686                                        page_length);
 687         kunmap_atomic(vaddr);
 688
 689         return ret;
 690 }
 691
 692 /* Only difference to the fast-path function is that this can handle bit17
 693  * and uses non-atomic copy and kmap functions. */
 694 static int
 695 shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 696                   char __user *user_data,
 697                   bool page_do_bit17_swizzling,
 698                   bool needs_clflush_before,
 699                   bool needs_clflush_after)
 700 {
 701         char *vaddr;
 702         int ret;
 703
 704         vaddr = kmap(page);
 705         if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
 706                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 707                                              page_length,
 708                                              page_do_bit17_swizzling);
 709         if (page_do_bit17_swizzling)
 710                 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
 711                                                 user_data,
 712                                                 page_length);
 713         else
 714                 ret = __copy_from_user(vaddr + shmem_page_offset,
 715                                        user_data,
 716                                        page_length);
 717         if (needs_clflush_after)
 718                 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
 719                                              page_length,
 720                                              page_do_bit17_swizzling);
 721         kunmap(page);
 722
 723         return ret;
 724 }
 725
 726 static int
 727 i915_gem_shmem_pwrite(struct drm_device *dev,
 728                       struct drm_i915_gem_object *obj,
 729                       struct drm_i915_gem_pwrite *args,
 730                       struct drm_file *file)
 731 {
 732         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 733         ssize_t remain;
 734         loff_t offset;
 735         char __user *user_data;
 736         int shmem_page_offset, page_length, ret = 0;
 737         int obj_do_bit17_swizzling, page_do_bit17_swizzling;
 738         int hit_slowpath = 0;
 739         int needs_clflush_after = 0;
 740         int needs_clflush_before = 0;
 741         int release_page;
 742
 743         user_data = (char __user *) (uintptr_t) args->data_ptr;
 744         remain = args->size;
 745
 746         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 747
 748         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 749                 /* If we're not in the cpu write domain, set ourself into the gtt
 750                  * write domain and manually flush cachelines (if required). This
 751                  * optimizes for the case when the gpu will use the data
 752                  * right away and we therefore have to clflush anyway. */
 753                 if (obj->cache_level == I915_CACHE_NONE)
 754                         needs_clflush_after = 1;
 755                 ret = i915_gem_object_set_to_gtt_domain(obj, true);
 756                 if (ret)
 757                         return ret;
 758         }
 759         /* Same trick applies for invalidate partially written cachelines before
 760          * writing.  */
 761         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
 762             && obj->cache_level == I915_CACHE_NONE)
 763                 needs_clflush_before = 1;
 764
 765         offset = args->offset;
 766         obj->dirty = 1;
 767
 768         while (remain > 0) {
 769                 struct page *page;
 770                 int partial_cacheline_write;
 771
 772                 /* Operation in this page
 773                  *
 774                  * shmem_page_offset = offset within page in shmem file
 775                  * page_length = bytes to copy for this page
 776                  */
 777                 shmem_page_offset = offset_in_page(offset);
 778
 779                 page_length = remain;
 780                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 781                         page_length = PAGE_SIZE - shmem_page_offset;
 782
 783                 /* If we don't overwrite a cacheline completely we need to be
 784                  * careful to have up-to-date data by first clflushing. Don't
 785                  * overcomplicate things and flush the entire patch. */
 786                 partial_cacheline_write = needs_clflush_before &&
 787                         ((shmem_page_offset | page_length)
 788                                 & (boot_cpu_data.x86_clflush_size - 1));
 789
 790                 if (obj->pages) {
 791                         page = obj->pages[offset >> PAGE_SHIFT];
 792                         release_page = 0;
 793                 } else {
 794                         page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
 795                         if (IS_ERR(page)) {
 796                                 ret = PTR_ERR(page);
 797                                 goto out;
 798                         }
 799                         release_page = 1;
 800                 }
 801
 802                 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
 803                         (page_to_phys(page) & (1 << 17)) != 0;
 804
 805                 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
 806                                         user_data, page_do_bit17_swizzling,
 807                                         partial_cacheline_write,
 808                                         needs_clflush_after);
 809                 if (ret == 0)
 810                         goto next_page;
 811
 812                 hit_slowpath = 1;
 813                 page_cache_get(page);
 814                 mutex_unlock(&dev->struct_mutex);
 815
 816                 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
 817                                         user_data, page_do_bit17_swizzling,
 818                                         partial_cacheline_write,
 819                                         needs_clflush_after);
 820
 821                 mutex_lock(&dev->struct_mutex);
 822                 page_cache_release(page);
 823 next_page:
 824                 set_page_dirty(page);
 825                 mark_page_accessed(page);
 826                 if (release_page)
 827                         page_cache_release(page);
 828
 829                 if (ret) {
 830                         ret = -EFAULT;
 831                         goto out;
 832                 }
 833
 834                 remain -= page_length;
 835                 user_data += page_length;
 836                 offset += page_length;
 837         }
 838
 839 out:
 840         if (hit_slowpath) {
 841                 /* Fixup: Kill any reinstated backing storage pages */
 842                 if (obj->madv == __I915_MADV_PURGED)
 843                         i915_gem_object_truncate(obj);
 844                 /* and flush dirty cachelines in case the object isn't in the cpu write
 845                  * domain anymore. */
 846                 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 847                         i915_gem_clflush_object(obj);
 848                         intel_gtt_chipset_flush();
 849                 }
 850         }
 851
 852         if (needs_clflush_after)
 853                 intel_gtt_chipset_flush();
 854
 855         return ret;
 856 }
 857
 858 /**
 859  * Writes data to the object referenced by handle.
 860  *
 861  * On error, the contents of the buffer that were to be modified are undefined.
 862  */
 863 int
 864 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 865                       struct drm_file *file)
 866 {
 867         struct drm_i915_gem_pwrite *args = data;
 868         struct drm_i915_gem_object *obj;
 869         int ret;
 870
 871         if (args->size == 0)
 872                 return 0;
 873
 874         if (!access_ok(VERIFY_READ,
 875                        (char __user *)(uintptr_t)args->data_ptr,
 876                        args->size))
 877                 return -EFAULT;
 878
 879         ret = fault_in_multipages_readable((char __user *)(uintptr_t)args->data_ptr,
 880                                            args->size);
 881         if (ret)
 882                 return -EFAULT;
 883
 884         ret = i915_mutex_lock_interruptible(dev);
 885         if (ret)
 886                 return ret;
 887
 888         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 889         if (&obj->base == NULL) {
 890                 ret = -ENOENT;
 891                 goto unlock;
 892         }
 893
 894         /* Bounds check destination. */
 895         if (args->offset > obj->base.size ||
 896             args->size > obj->base.size - args->offset) {
 897                 ret = -EINVAL;
 898                 goto out;
 899         }
 900
 901         /* prime objects have no backing filp to GEM pread/pwrite
 902          * pages from.
 903          */
 904         if (!obj->base.filp) {
 905                 ret = -EINVAL;
 906                 goto out;
 907         }
 908
 909         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 910
 911         ret = -EFAULT;
 912         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 913          * it would end up going through the fenced access, and we'll get
 914          * different detiling behavior between reading and writing.
 915          * pread/pwrite currently are reading and writing from the CPU
 916          * perspective, requiring manual detiling by the client.
 917          */
 918         if (obj->phys_obj) {
 919                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
 920                 goto out;
 921         }
 922
 923         if (obj->gtt_space &&
 924             obj->cache_level == I915_CACHE_NONE &&
 925             obj->tiling_mode == I915_TILING_NONE &&
 926             obj->map_and_fenceable &&
 927             obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
 928                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
 929                 /* Note that the gtt paths might fail with non-page-backed user
 930                  * pointers (e.g. gtt mappings when moving data between
 931                  * textures). Fallback to the shmem path in that case. */
 932         }
 933
 934         if (ret == -EFAULT)
 935                 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
 936
 937 out:
 938         drm_gem_object_unreference(&obj->base);
 939 unlock:
 940         mutex_unlock(&dev->struct_mutex);
 941         return ret;
 942 }
 943
 944 /**
 945  * Called when user space prepares to use an object with the CPU, either
 946  * through the mmap ioctl's mapping or a GTT mapping.
 947  */
 948 int
 949 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 950                           struct drm_file *file)
 951 {
 952         struct drm_i915_gem_set_domain *args = data;
 953         struct drm_i915_gem_object *obj;
 954         uint32_t read_domains = args->read_domains;
 955         uint32_t write_domain = args->write_domain;
 956         int ret;
 957
 958         /* Only handle setting domains to types used by the CPU. */
 959         if (write_domain & I915_GEM_GPU_DOMAINS)
 960                 return -EINVAL;
 961
 962         if (read_domains & I915_GEM_GPU_DOMAINS)
 963                 return -EINVAL;
 964
 965         /* Having something in the write domain implies it's in the read
 966          * domain, and only that read domain.  Enforce that in the request.
 967          */
 968         if (write_domain != 0 && read_domains != write_domain)
 969                 return -EINVAL;
 970
 971         ret = i915_mutex_lock_interruptible(dev);
 972         if (ret)
 973                 return ret;
 974
 975         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 976         if (&obj->base == NULL) {
 977                 ret = -ENOENT;
 978                 goto unlock;
 979         }
 980
 981         if (read_domains & I915_GEM_DOMAIN_GTT) {
 982                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 983
 984                 /* Silently promote "you're not bound, there was nothing to do"
 985                  * to success, since the client was just asking us to
 986                  * make sure everything was done.
 987                  */
 988                 if (ret == -EINVAL)
 989                         ret = 0;
 990         } else {
 991                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 992         }
 993
 994         drm_gem_object_unreference(&obj->base);
 995 unlock:
 996         mutex_unlock(&dev->struct_mutex);
 997         return ret;
 998 }
 999
1000 /**
1001  * Called when user space has done writes to this buffer
1002  */
1003 int
1004 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1005                          struct drm_file *file)
1006 {
1007         struct drm_i915_gem_sw_finish *args = data;
1008         struct drm_i915_gem_object *obj;
1009         int ret = 0;
1010
1011         ret = i915_mutex_lock_interruptible(dev);
1012         if (ret)
1013                 return ret;
1014
1015         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1016         if (&obj->base == NULL) {
1017                 ret = -ENOENT;
1018                 goto unlock;
1019         }
1020
1021         /* Pinned buffers may be scanout, so flush the cache */
1022         if (obj->pin_count)
1023                 i915_gem_object_flush_cpu_write_domain(obj);
1024
1025         drm_gem_object_unreference(&obj->base);
1026 unlock:
1027         mutex_unlock(&dev->struct_mutex);
1028         return ret;
1029 }
1030
1031 /**
1032  * Maps the contents of an object, returning the address it is mapped
1033  * into.
1034  *
1035  * While the mapping holds a reference on the contents of the object, it doesn't
1036  * imply a ref on the object itself.
1037  */
1038 int
1039 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1040                     struct drm_file *file)
1041 {
1042         struct drm_i915_gem_mmap *args = data;
1043         struct drm_gem_object *obj;
1044         unsigned long addr;
1045
1046         obj = drm_gem_object_lookup(dev, file, args->handle);
1047         if (obj == NULL)
1048                 return -ENOENT;
1049
1050         /* prime objects have no backing filp to GEM mmap
1051          * pages from.
1052          */
1053         if (!obj->filp) {
1054                 drm_gem_object_unreference_unlocked(obj);
1055                 return -EINVAL;
1056         }
1057
1058         addr = vm_mmap(obj->filp, 0, args->size,
1059                        PROT_READ | PROT_WRITE, MAP_SHARED,
1060                        args->offset);
1061         drm_gem_object_unreference_unlocked(obj);
1062         if (IS_ERR((void *)addr))
1063                 return addr;
1064
1065         args->addr_ptr = (uint64_t) addr;
1066
1067         return 0;
1068 }
1069
1070 /**
1071  * i915_gem_fault - fault a page into the GTT
1072  * vma: VMA in question
1073  * vmf: fault info
1074  *
1075  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1076  * from userspace.  The fault handler takes care of binding the object to
1077  * the GTT (if needed), allocating and programming a fence register (again,
1078  * only if needed based on whether the old reg is still valid or the object
1079  * is tiled) and inserting a new PTE into the faulting process.
1080  *
1081  * Note that the faulting process may involve evicting existing objects
1082  * from the GTT and/or fence registers to make room.  So performance may
1083  * suffer if the GTT working set is large or there are few fence registers
1084  * left.
1085  */
1086 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1087 {
1088         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1089         struct drm_device *dev = obj->base.dev;
1090         drm_i915_private_t *dev_priv = dev->dev_private;
1091         pgoff_t page_offset;
1092         unsigned long pfn;
1093         int ret = 0;
1094         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1095
1096         /* We don't use vmf->pgoff since that has the fake offset */
1097         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1098                 PAGE_SHIFT;
1099
1100         ret = i915_mutex_lock_interruptible(dev);
1101         if (ret)
1102                 goto out;
1103
1104         trace_i915_gem_object_fault(obj, page_offset, true, write);
1105
1106         /* Now bind it into the GTT if needed */
1107         if (!obj->map_and_fenceable) {
1108                 ret = i915_gem_object_unbind(obj);
1109                 if (ret)
1110                         goto unlock;
1111         }
1112         if (!obj->gtt_space) {
1113                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1114                 if (ret)
1115                         goto unlock;
1116
1117                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1118                 if (ret)
1119                         goto unlock;
1120         }
1121
1122         if (!obj->has_global_gtt_mapping)
1123                 i915_gem_gtt_bind_object(obj, obj->cache_level);
1124
1125         ret = i915_gem_object_get_fence(obj);
1126         if (ret)
1127                 goto unlock;
1128
1129         if (i915_gem_object_is_inactive(obj))
1130                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1131
1132         obj->fault_mappable = true;
1133
1134         pfn = ((dev_priv->mm.gtt_base_addr + obj->gtt_offset) >> PAGE_SHIFT) +
1135                 page_offset;
1136
1137         /* Finally, remap it using the new GTT offset */
1138         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1139 unlock:
1140         mutex_unlock(&dev->struct_mutex);
1141 out:
1142         switch (ret) {
1143         case -EIO:
1144                 /* If this -EIO is due to a gpu hang, give the reset code a
1145                  * chance to clean up the mess. Otherwise return the proper
1146                  * SIGBUS. */
1147                 if (!atomic_read(&dev_priv->mm.wedged))
1148                         return VM_FAULT_SIGBUS;
1149         case -EAGAIN:
1150                 /* Give the error handler a chance to run and move the
1151                  * objects off the GPU active list. Next time we service the
1152                  * fault, we should be able to transition the page into the
1153                  * GTT without touching the GPU (and so avoid further
1154                  * EIO/EGAIN). If the GPU is wedged, then there is no issue
1155                  * with coherency, just lost writes.
1156                  */
1157                 set_need_resched();
1158         case 0:
1159         case -ERESTARTSYS:
1160         case -EINTR:
1161                 return VM_FAULT_NOPAGE;
1162         case -ENOMEM:
1163                 return VM_FAULT_OOM;
1164         default:
1165                 return VM_FAULT_SIGBUS;
1166         }
1167 }
1168
1169 /**
1170  * i915_gem_release_mmap - remove physical page mappings
1171  * @obj: obj in question
1172  *
1173  * Preserve the reservation of the mmapping with the DRM core code, but
1174  * relinquish ownership of the pages back to the system.
1175  *
1176  * It is vital that we remove the page mapping if we have mapped a tiled
1177  * object through the GTT and then lose the fence register due to
1178  * resource pressure. Similarly if the object has been moved out of the
1179  * aperture, than pages mapped into userspace must be revoked. Removing the
1180  * mapping will then trigger a page fault on the next user access, allowing
1181  * fixup by i915_gem_fault().
1182  */
1183 void
1184 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1185 {
1186         if (!obj->fault_mappable)
1187                 return;
1188
1189         if (obj->base.dev->dev_mapping)
1190                 unmap_mapping_range(obj->base.dev->dev_mapping,
1191                                     (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1192                                     obj->base.size, 1);
1193
1194         obj->fault_mappable = false;
1195 }
1196
1197 static uint32_t
1198 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1199 {
1200         uint32_t gtt_size;
1201
1202         if (INTEL_INFO(dev)->gen >= 4 ||
1203             tiling_mode == I915_TILING_NONE)
1204                 return size;
1205
1206         /* Previous chips need a power-of-two fence region when tiling */
1207         if (INTEL_INFO(dev)->gen == 3)
1208                 gtt_size = 1024*1024;
1209         else
1210                 gtt_size = 512*1024;
1211
1212         while (gtt_size < size)
1213                 gtt_size <<= 1;
1214
1215         return gtt_size;
1216 }
1217
1218 /**
1219  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1220  * @obj: object to check
1221  *
1222  * Return the required GTT alignment for an object, taking into account
1223  * potential fence register mapping.
1224  */
1225 static uint32_t
1226 i915_gem_get_gtt_alignment(struct drm_device *dev,
1227                            uint32_t size,
1228                            int tiling_mode)
1229 {
1230         /*
1231          * Minimum alignment is 4k (GTT page size), but might be greater
1232          * if a fence register is needed for the object.
1233          */
1234         if (INTEL_INFO(dev)->gen >= 4 ||
1235             tiling_mode == I915_TILING_NONE)
1236                 return 4096;
1237
1238         /*
1239          * Previous chips need to be aligned to the size of the smallest
1240          * fence register that can contain the object.
1241          */
1242         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1243 }
1244
1245 /**
1246  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1247  *                                       unfenced object
1248  * @dev: the device
1249  * @size: size of the object
1250  * @tiling_mode: tiling mode of the object
1251  *
1252  * Return the required GTT alignment for an object, only taking into account
1253  * unfenced tiled surface requirements.
1254  */
1255 uint32_t
1256 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1257                                     uint32_t size,
1258                                     int tiling_mode)
1259 {
1260         /*
1261          * Minimum alignment is 4k (GTT page size) for sane hw.
1262          */
1263         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1264             tiling_mode == I915_TILING_NONE)
1265                 return 4096;
1266
1267         /* Previous hardware however needs to be aligned to a power-of-two
1268          * tile height. The simplest method for determining this is to reuse
1269          * the power-of-tile object size.
1270          */
1271         return i915_gem_get_gtt_size(dev, size, tiling_mode);
1272 }
1273
1274 int
1275 i915_gem_mmap_gtt(struct drm_file *file,
1276                   struct drm_device *dev,
1277                   uint32_t handle,
1278                   uint64_t *offset)
1279 {
1280         struct drm_i915_private *dev_priv = dev->dev_private;
1281         struct drm_i915_gem_object *obj;
1282         int ret;
1283
1284         ret = i915_mutex_lock_interruptible(dev);
1285         if (ret)
1286                 return ret;
1287
1288         obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1289         if (&obj->base == NULL) {
1290                 ret = -ENOENT;
1291                 goto unlock;
1292         }
1293
1294         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1295                 ret = -E2BIG;
1296                 goto out;
1297         }
1298
1299         if (obj->madv != I915_MADV_WILLNEED) {
1300                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1301                 ret = -EINVAL;
1302                 goto out;
1303         }
1304
1305         if (!obj->base.map_list.map) {
1306                 ret = drm_gem_create_mmap_offset(&obj->base);
1307                 if (ret)
1308                         goto out;
1309         }
1310
1311         *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1312
1313 out:
1314         drm_gem_object_unreference(&obj->base);
1315 unlock:
1316         mutex_unlock(&dev->struct_mutex);
1317         return ret;
1318 }
1319
1320 /**
1321  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1322  * @dev: DRM device
1323  * @data: GTT mapping ioctl data
1324  * @file: GEM object info
1325  *
1326  * Simply returns the fake offset to userspace so it can mmap it.
1327  * The mmap call will end up in drm_gem_mmap(), which will set things
1328  * up so we can get faults in the handler above.
1329  *
1330  * The fault handler will take care of binding the object into the GTT
1331  * (since it may have been evicted to make room for something), allocating
1332  * a fence register, and mapping the appropriate aperture address into
1333  * userspace.
1334  */
1335 int
1336 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1337                         struct drm_file *file)
1338 {
1339         struct drm_i915_gem_mmap_gtt *args = data;
1340
1341         return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1342 }
1343
1344 int
1345 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1346                               gfp_t gfpmask)
1347 {
1348         int page_count, i;
1349         struct address_space *mapping;
1350         struct inode *inode;
1351         struct page *page;
1352
1353         if (obj->pages || obj->sg_table)
1354                 return 0;
1355
1356         /* Get the list of pages out of our struct file.  They'll be pinned
1357          * at this point until we release them.
1358          */
1359         page_count = obj->base.size / PAGE_SIZE;
1360         BUG_ON(obj->pages != NULL);
1361         obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1362         if (obj->pages == NULL)
1363                 return -ENOMEM;
1364
1365         inode = obj->base.filp->f_path.dentry->d_inode;
1366         mapping = inode->i_mapping;
1367         gfpmask |= mapping_gfp_mask(mapping);
1368
1369         for (i = 0; i < page_count; i++) {
1370                 page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
1371                 if (IS_ERR(page))
1372                         goto err_pages;
1373
1374                 obj->pages[i] = page;
1375         }
1376
1377         if (i915_gem_object_needs_bit17_swizzle(obj))
1378                 i915_gem_object_do_bit_17_swizzle(obj);
1379
1380         return 0;
1381
1382 err_pages:
1383         while (i--)
1384                 page_cache_release(obj->pages[i]);
1385
1386         drm_free_large(obj->pages);
1387         obj->pages = NULL;
1388         return PTR_ERR(page);
1389 }
1390
1391 static void
1392 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1393 {
1394         int page_count = obj->base.size / PAGE_SIZE;
1395         int i;
1396
1397         if (!obj->pages)
1398                 return;
1399
1400         BUG_ON(obj->madv == __I915_MADV_PURGED);
1401
1402         if (i915_gem_object_needs_bit17_swizzle(obj))
1403                 i915_gem_object_save_bit_17_swizzle(obj);
1404
1405         if (obj->madv == I915_MADV_DONTNEED)
1406                 obj->dirty = 0;
1407
1408         for (i = 0; i < page_count; i++) {
1409                 if (obj->dirty)
1410                         set_page_dirty(obj->pages[i]);
1411
1412                 if (obj->madv == I915_MADV_WILLNEED)
1413                         mark_page_accessed(obj->pages[i]);
1414
1415                 page_cache_release(obj->pages[i]);
1416         }
1417         obj->dirty = 0;
1418
1419         drm_free_large(obj->pages);
1420         obj->pages = NULL;
1421 }
1422
1423 void
1424 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1425                                struct intel_ring_buffer *ring,
1426                                u32 seqno)
1427 {
1428         struct drm_device *dev = obj->base.dev;
1429         struct drm_i915_private *dev_priv = dev->dev_private;
1430
1431         BUG_ON(ring == NULL);
1432         obj->ring = ring;
1433
1434         /* Add a reference if we're newly entering the active list. */
1435         if (!obj->active) {
1436                 drm_gem_object_reference(&obj->base);
1437                 obj->active = 1;
1438         }
1439
1440         /* Move from whatever list we were on to the tail of execution. */
1441         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1442         list_move_tail(&obj->ring_list, &ring->active_list);
1443
1444         obj->last_read_seqno = seqno;
1445
1446         if (obj->fenced_gpu_access) {
1447                 obj->last_fenced_seqno = seqno;
1448
1449                 /* Bump MRU to take account of the delayed flush */
1450                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1451                         struct drm_i915_fence_reg *reg;
1452
1453                         reg = &dev_priv->fence_regs[obj->fence_reg];
1454                         list_move_tail(&reg->lru_list,
1455                                        &dev_priv->mm.fence_list);
1456                 }
1457         }
1458 }
1459
1460 static void
1461 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1462 {
1463         struct drm_device *dev = obj->base.dev;
1464         struct drm_i915_private *dev_priv = dev->dev_private;
1465
1466         list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1467
1468         BUG_ON(!list_empty(&obj->gpu_write_list));
1469         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
1470         BUG_ON(!obj->active);
1471
1472         list_del_init(&obj->ring_list);
1473         obj->ring = NULL;
1474
1475         obj->last_read_seqno = 0;
1476         obj->last_write_seqno = 0;
1477         obj->base.write_domain = 0;
1478
1479         obj->last_fenced_seqno = 0;
1480         obj->fenced_gpu_access = false;
1481
1482         obj->active = 0;
1483         drm_gem_object_unreference(&obj->base);
1484
1485         WARN_ON(i915_verify_lists(dev));
1486 }
1487
1488 /* Immediately discard the backing storage */
1489 static void
1490 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1491 {
1492         struct inode *inode;
1493
1494         /* Our goal here is to return as much of the memory as
1495          * is possible back to the system as we are called from OOM.
1496          * To do this we must instruct the shmfs to drop all of its
1497          * backing pages, *now*.
1498          */
1499         inode = obj->base.filp->f_path.dentry->d_inode;
1500         shmem_truncate_range(inode, 0, (loff_t)-1);
1501
1502         if (obj->base.map_list.map)
1503                 drm_gem_free_mmap_offset(&obj->base);
1504
1505         obj->madv = __I915_MADV_PURGED;
1506 }
1507
1508 static inline int
1509 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1510 {
1511         return obj->madv == I915_MADV_DONTNEED;
1512 }
1513
1514 static void
1515 i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1516                                uint32_t flush_domains)
1517 {
1518         struct drm_i915_gem_object *obj, *next;
1519
1520         list_for_each_entry_safe(obj, next,
1521                                  &ring->gpu_write_list,
1522                                  gpu_write_list) {
1523                 if (obj->base.write_domain & flush_domains) {
1524                         uint32_t old_write_domain = obj->base.write_domain;
1525
1526                         obj->base.write_domain = 0;
1527                         list_del_init(&obj->gpu_write_list);
1528                         i915_gem_object_move_to_active(obj, ring,
1529                                                        i915_gem_next_request_seqno(ring));
1530
1531                         trace_i915_gem_object_change_domain(obj,
1532                                                             obj->base.read_domains,
1533                                                             old_write_domain);
1534                 }
1535         }
1536 }
1537
1538 static u32
1539 i915_gem_get_seqno(struct drm_device *dev)
1540 {
1541         drm_i915_private_t *dev_priv = dev->dev_private;
1542         u32 seqno = dev_priv->next_seqno;
1543
1544         /* reserve 0 for non-seqno */
1545         if (++dev_priv->next_seqno == 0)
1546                 dev_priv->next_seqno = 1;
1547
1548         return seqno;
1549 }
1550
1551 u32
1552 i915_gem_next_request_seqno(struct intel_ring_buffer *ring)
1553 {
1554         if (ring->outstanding_lazy_request == 0)
1555                 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev);
1556
1557         return ring->outstanding_lazy_request;
1558 }
1559
1560 int
1561 i915_add_request(struct intel_ring_buffer *ring,
1562                  struct drm_file *file,
1563                  struct drm_i915_gem_request *request)
1564 {
1565         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1566         uint32_t seqno;
1567         u32 request_ring_position;
1568         int was_empty;
1569         int ret;
1570
1571         /*
1572          * Emit any outstanding flushes - execbuf can fail to emit the flush
1573          * after having emitted the batchbuffer command. Hence we need to fix
1574          * things up similar to emitting the lazy request. The difference here
1575          * is that the flush _must_ happen before the next request, no matter
1576          * what.
1577          */
1578         if (ring->gpu_caches_dirty) {
1579                 ret = i915_gem_flush_ring(ring, 0, I915_GEM_GPU_DOMAINS);
1580                 if (ret)
1581                         return ret;
1582
1583                 ring->gpu_caches_dirty = false;
1584         }
1585
1586         if (request == NULL) {
1587                 request = kmalloc(sizeof(*request), GFP_KERNEL);
1588                 if (request == NULL)
1589                         return -ENOMEM;
1590         }
1591
1592         seqno = i915_gem_next_request_seqno(ring);
1593
1594         /* Record the position of the start of the request so that
1595          * should we detect the updated seqno part-way through the
1596          * GPU processing the request, we never over-estimate the
1597          * position of the head.
1598          */
1599         request_ring_position = intel_ring_get_tail(ring);
1600
1601         ret = ring->add_request(ring, &seqno);
1602         if (ret) {
1603                 kfree(request);
1604                 return ret;
1605         }
1606
1607         trace_i915_gem_request_add(ring, seqno);
1608
1609         request->seqno = seqno;
1610         request->ring = ring;
1611         request->tail = request_ring_position;
1612         request->emitted_jiffies = jiffies;
1613         was_empty = list_empty(&ring->request_list);
1614         list_add_tail(&request->list, &ring->request_list);
1615         request->file_priv = NULL;
1616
1617         if (file) {
1618                 struct drm_i915_file_private *file_priv = file->driver_priv;
1619
1620                 spin_lock(&file_priv->mm.lock);
1621                 request->file_priv = file_priv;
1622                 list_add_tail(&request->client_list,
1623                               &file_priv->mm.request_list);
1624                 spin_unlock(&file_priv->mm.lock);
1625         }
1626
1627         ring->outstanding_lazy_request = 0;
1628
1629         if (!dev_priv->mm.suspended) {
1630                 if (i915_enable_hangcheck) {
1631                         mod_timer(&dev_priv->hangcheck_timer,
1632                                   jiffies +
1633                                   msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1634                 }
1635                 if (was_empty)
1636                         queue_delayed_work(dev_priv->wq,
1637                                            &dev_priv->mm.retire_work, HZ);
1638         }
1639
1640         WARN_ON(!list_empty(&ring->gpu_write_list));
1641
1642         return 0;
1643 }
1644
1645 static inline void
1646 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1647 {
1648         struct drm_i915_file_private *file_priv = request->file_priv;
1649
1650         if (!file_priv)
1651                 return;
1652
1653         spin_lock(&file_priv->mm.lock);
1654         if (request->file_priv) {
1655                 list_del(&request->client_list);
1656                 request->file_priv = NULL;
1657         }
1658         spin_unlock(&file_priv->mm.lock);
1659 }
1660
1661 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1662                                       struct intel_ring_buffer *ring)
1663 {
1664         while (!list_empty(&ring->request_list)) {
1665                 struct drm_i915_gem_request *request;
1666
1667                 request = list_first_entry(&ring->request_list,
1668                                            struct drm_i915_gem_request,
1669                                            list);
1670
1671                 list_del(&request->list);
1672                 i915_gem_request_remove_from_client(request);
1673                 kfree(request);
1674         }
1675
1676         while (!list_empty(&ring->active_list)) {
1677                 struct drm_i915_gem_object *obj;
1678
1679                 obj = list_first_entry(&ring->active_list,
1680                                        struct drm_i915_gem_object,
1681                                        ring_list);
1682
1683                 list_del_init(&obj->gpu_write_list);
1684                 i915_gem_object_move_to_inactive(obj);
1685         }
1686 }
1687
1688 static void i915_gem_reset_fences(struct drm_device *dev)
1689 {
1690         struct drm_i915_private *dev_priv = dev->dev_private;
1691         int i;
1692
1693         for (i = 0; i < dev_priv->num_fence_regs; i++) {
1694                 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1695
1696                 i915_gem_write_fence(dev, i, NULL);
1697
1698                 if (reg->obj)
1699                         i915_gem_object_fence_lost(reg->obj);
1700
1701                 reg->pin_count = 0;
1702                 reg->obj = NULL;
1703                 INIT_LIST_HEAD(&reg->lru_list);
1704         }
1705
1706         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
1707 }
1708
1709 void i915_gem_reset(struct drm_device *dev)
1710 {
1711         struct drm_i915_private *dev_priv = dev->dev_private;
1712         struct drm_i915_gem_object *obj;
1713         struct intel_ring_buffer *ring;
1714         int i;
1715
1716         for_each_ring(ring, dev_priv, i)
1717                 i915_gem_reset_ring_lists(dev_priv, ring);
1718
1719         /* Move everything out of the GPU domains to ensure we do any
1720          * necessary invalidation upon reuse.
1721          */
1722         list_for_each_entry(obj,
1723                             &dev_priv->mm.inactive_list,
1724                             mm_list)
1725         {
1726                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1727         }
1728
1729         /* The fence registers are invalidated so clear them out */
1730         i915_gem_reset_fences(dev);
1731 }
1732
1733 /**
1734  * This function clears the request list as sequence numbers are passed.
1735  */
1736 void
1737 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1738 {
1739         uint32_t seqno;
1740         int i;
1741
1742         if (list_empty(&ring->request_list))
1743                 return;
1744
1745         WARN_ON(i915_verify_lists(ring->dev));
1746
1747         seqno = ring->get_seqno(ring);
1748
1749         for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1750                 if (seqno >= ring->sync_seqno[i])
1751                         ring->sync_seqno[i] = 0;
1752
1753         while (!list_empty(&ring->request_list)) {
1754                 struct drm_i915_gem_request *request;
1755
1756                 request = list_first_entry(&ring->request_list,
1757                                            struct drm_i915_gem_request,
1758                                            list);
1759
1760                 if (!i915_seqno_passed(seqno, request->seqno))
1761                         break;
1762
1763                 trace_i915_gem_request_retire(ring, request->seqno);
1764                 /* We know the GPU must have read the request to have
1765                  * sent us the seqno + interrupt, so use the position
1766                  * of tail of the request to update the last known position
1767                  * of the GPU head.
1768                  */
1769                 ring->last_retired_head = request->tail;
1770
1771                 list_del(&request->list);
1772                 i915_gem_request_remove_from_client(request);
1773                 kfree(request);
1774         }
1775
1776         /* Move any buffers on the active list that are no longer referenced
1777          * by the ringbuffer to the flushing/inactive lists as appropriate.
1778          */
1779         while (!list_empty(&ring->active_list)) {
1780                 struct drm_i915_gem_object *obj;
1781
1782                 obj = list_first_entry(&ring->active_list,
1783                                       struct drm_i915_gem_object,
1784                                       ring_list);
1785
1786                 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
1787                         break;
1788
1789                 i915_gem_object_move_to_inactive(obj);
1790         }
1791
1792         if (unlikely(ring->trace_irq_seqno &&
1793                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1794                 ring->irq_put(ring);
1795                 ring->trace_irq_seqno = 0;
1796         }
1797
1798         WARN_ON(i915_verify_lists(ring->dev));
1799 }
1800
1801 void
1802 i915_gem_retire_requests(struct drm_device *dev)
1803 {
1804         drm_i915_private_t *dev_priv = dev->dev_private;
1805         struct intel_ring_buffer *ring;
1806         int i;
1807
1808         for_each_ring(ring, dev_priv, i)
1809                 i915_gem_retire_requests_ring(ring);
1810 }
1811
1812 static void
1813 i915_gem_retire_work_handler(struct work_struct *work)
1814 {
1815         drm_i915_private_t *dev_priv;
1816         struct drm_device *dev;
1817         struct intel_ring_buffer *ring;
1818         bool idle;
1819         int i;
1820
1821         dev_priv = container_of(work, drm_i915_private_t,
1822                                 mm.retire_work.work);
1823         dev = dev_priv->dev;
1824
1825         /* Come back later if the device is busy... */
1826         if (!mutex_trylock(&dev->struct_mutex)) {
1827                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1828                 return;
1829         }
1830
1831         i915_gem_retire_requests(dev);
1832
1833         /* Send a periodic flush down the ring so we don't hold onto GEM
1834          * objects indefinitely.
1835          */
1836         idle = true;
1837         for_each_ring(ring, dev_priv, i) {
1838                 if (ring->gpu_caches_dirty)
1839                         i915_add_request(ring, NULL, NULL);
1840
1841                 idle &= list_empty(&ring->request_list);
1842         }
1843
1844         if (!dev_priv->mm.suspended && !idle)
1845                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1846
1847         mutex_unlock(&dev->struct_mutex);
1848 }
1849
1850 int
1851 i915_gem_check_wedge(struct drm_i915_private *dev_priv,
1852                      bool interruptible)
1853 {
1854         if (atomic_read(&dev_priv->mm.wedged)) {
1855                 struct completion *x = &dev_priv->error_completion;
1856                 bool recovery_complete;
1857                 unsigned long flags;
1858
1859                 /* Give the error handler a chance to run. */
1860                 spin_lock_irqsave(&x->wait.lock, flags);
1861                 recovery_complete = x->done > 0;
1862                 spin_unlock_irqrestore(&x->wait.lock, flags);
1863
1864                 /* Non-interruptible callers can't handle -EAGAIN, hence return
1865                  * -EIO unconditionally for these. */
1866                 if (!interruptible)
1867                         return -EIO;
1868
1869                 /* Recovery complete, but still wedged means reset failure. */
1870                 if (recovery_complete)
1871                         return -EIO;
1872
1873                 return -EAGAIN;
1874         }
1875
1876         return 0;
1877 }
1878
1879 /*
1880  * Compare seqno against outstanding lazy request. Emit a request if they are
1881  * equal.
1882  */
1883 static int
1884 i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
1885 {
1886         int ret;
1887
1888         BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
1889
1890         ret = 0;
1891         if (seqno == ring->outstanding_lazy_request)
1892                 ret = i915_add_request(ring, NULL, NULL);
1893
1894         return ret;
1895 }
1896
1897 /**
1898  * __wait_seqno - wait until execution of seqno has finished
1899  * @ring: the ring expected to report seqno
1900  * @seqno: duh!
1901  * @interruptible: do an interruptible wait (normally yes)
1902  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1903  *
1904  * Returns 0 if the seqno was found within the alloted time. Else returns the
1905  * errno with remaining time filled in timeout argument.
1906  */
1907 static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
1908                         bool interruptible, struct timespec *timeout)
1909 {
1910         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1911         struct timespec before, now, wait_time={1,0};
1912         unsigned long timeout_jiffies;
1913         long end;
1914         bool wait_forever = true;
1915         int ret;
1916
1917         if (i915_seqno_passed(ring->get_seqno(ring), seqno))
1918                 return 0;
1919
1920         trace_i915_gem_request_wait_begin(ring, seqno);
1921
1922         if (timeout != NULL) {
1923                 wait_time = *timeout;
1924                 wait_forever = false;
1925         }
1926
1927         timeout_jiffies = timespec_to_jiffies(&wait_time);
1928
1929         if (WARN_ON(!ring->irq_get(ring)))
1930                 return -ENODEV;
1931
1932         /* Record current time in case interrupted by signal, or wedged * */
1933         getrawmonotonic(&before);
1934
1935 #define EXIT_COND \
1936         (i915_seqno_passed(ring->get_seqno(ring), seqno) || \
1937         atomic_read(&dev_priv->mm.wedged))
1938         do {
1939                 if (interruptible)
1940                         end = wait_event_interruptible_timeout(ring->irq_queue,
1941                                                                EXIT_COND,
1942                                                                timeout_jiffies);
1943                 else
1944                         end = wait_event_timeout(ring->irq_queue, EXIT_COND,
1945                                                  timeout_jiffies);
1946
1947                 ret = i915_gem_check_wedge(dev_priv, interruptible);
1948                 if (ret)
1949                         end = ret;
1950         } while (end == 0 && wait_forever);
1951
1952         getrawmonotonic(&now);
1953
1954         ring->irq_put(ring);
1955         trace_i915_gem_request_wait_end(ring, seqno);
1956 #undef EXIT_COND
1957
1958         if (timeout) {
1959                 struct timespec sleep_time = timespec_sub(now, before);
1960                 *timeout = timespec_sub(*timeout, sleep_time);
1961         }
1962
1963         switch (end) {
1964         case -EIO:
1965         case -EAGAIN: /* Wedged */
1966         case -ERESTARTSYS: /* Signal */
1967                 return (int)end;
1968         case 0: /* Timeout */
1969                 if (timeout)
1970                         set_normalized_timespec(timeout, 0, 0);
1971                 return -ETIME;
1972         default: /* Completed */
1973                 WARN_ON(end < 0); /* We're not aware of other errors */
1974                 return 0;
1975         }
1976 }
1977
1978 /**
1979  * Waits for a sequence number to be signaled, and cleans up the
1980  * request and object lists appropriately for that event.
1981  */
1982 int
1983 i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
1984 {
1985         drm_i915_private_t *dev_priv = ring->dev->dev_private;
1986         int ret = 0;
1987
1988         BUG_ON(seqno == 0);
1989
1990         ret = i915_gem_check_wedge(dev_priv, dev_priv->mm.interruptible);
1991         if (ret)
1992                 return ret;
1993
1994         ret = i915_gem_check_olr(ring, seqno);
1995         if (ret)
1996                 return ret;
1997
1998         ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible, NULL);
1999
2000         return ret;
2001 }
2002
2003 /**
2004  * Ensures that all rendering to the object has completed and the object is
2005  * safe to unbind from the GTT or access from the CPU.
2006  */
2007 static __must_check int
2008 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
2009                                bool readonly)
2010 {
2011         u32 seqno;
2012         int ret;
2013
2014         /* This function only exists to support waiting for existing rendering,
2015          * not for emitting required flushes.
2016          */
2017         BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2018
2019         /* If there is rendering queued on the buffer being evicted, wait for
2020          * it.
2021          */
2022         if (readonly)
2023                 seqno = obj->last_write_seqno;
2024         else
2025                 seqno = obj->last_read_seqno;
2026         if (seqno == 0)
2027                 return 0;
2028
2029         ret = i915_wait_seqno(obj->ring, seqno);
2030         if (ret)
2031                 return ret;
2032
2033         /* Manually manage the write flush as we may have not yet retired
2034          * the buffer.
2035          */
2036         if (obj->last_write_seqno &&
2037             i915_seqno_passed(seqno, obj->last_write_seqno)) {
2038                 obj->last_write_seqno = 0;
2039                 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
2040         }
2041
2042         i915_gem_retire_requests_ring(obj->ring);
2043         return 0;
2044 }
2045
2046 /**
2047  * Ensures that an object will eventually get non-busy by flushing any required
2048  * write domains, emitting any outstanding lazy request and retiring and
2049  * completed requests.
2050  */
2051 static int
2052 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2053 {
2054         int ret;
2055
2056         if (obj->active) {
2057                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2058                 if (ret)
2059                         return ret;
2060
2061                 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
2062                 if (ret)
2063                         return ret;
2064
2065                 i915_gem_retire_requests_ring(obj->ring);
2066         }
2067
2068         return 0;
2069 }
2070
2071 /**
2072  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2073  * @DRM_IOCTL_ARGS: standard ioctl arguments
2074  *
2075  * Returns 0 if successful, else an error is returned with the remaining time in
2076  * the timeout parameter.
2077  *  -ETIME: object is still busy after timeout
2078  *  -ERESTARTSYS: signal interrupted the wait
2079  *  -ENONENT: object doesn't exist
2080  * Also possible, but rare:
2081  *  -EAGAIN: GPU wedged
2082  *  -ENOMEM: damn
2083  *  -ENODEV: Internal IRQ fail
2084  *  -E?: The add request failed
2085  *
2086  * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2087  * non-zero timeout parameter the wait ioctl will wait for the given number of
2088  * nanoseconds on an object becoming unbusy. Since the wait itself does so
2089  * without holding struct_mutex the object may become re-busied before this
2090  * function completes. A similar but shorter * race condition exists in the busy
2091  * ioctl
2092  */
2093 int
2094 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2095 {
2096         struct drm_i915_gem_wait *args = data;
2097         struct drm_i915_gem_object *obj;
2098         struct intel_ring_buffer *ring = NULL;
2099         struct timespec timeout_stack, *timeout = NULL;
2100         u32 seqno = 0;
2101         int ret = 0;
2102
2103         if (args->timeout_ns >= 0) {
2104                 timeout_stack = ns_to_timespec(args->timeout_ns);
2105                 timeout = &timeout_stack;
2106         }
2107
2108         ret = i915_mutex_lock_interruptible(dev);
2109         if (ret)
2110                 return ret;
2111
2112         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2113         if (&obj->base == NULL) {
2114                 mutex_unlock(&dev->struct_mutex);
2115                 return -ENOENT;
2116         }
2117
2118         /* Need to make sure the object gets inactive eventually. */
2119         ret = i915_gem_object_flush_active(obj);
2120         if (ret)
2121                 goto out;
2122
2123         if (obj->active) {
2124                 seqno = obj->last_read_seqno;
2125                 ring = obj->ring;
2126         }
2127
2128         if (seqno == 0)
2129                  goto out;
2130
2131         /* Do this after OLR check to make sure we make forward progress polling
2132          * on this IOCTL with a 0 timeout (like busy ioctl)
2133          */
2134         if (!args->timeout_ns) {
2135                 ret = -ETIME;
2136                 goto out;
2137         }
2138
2139         drm_gem_object_unreference(&obj->base);
2140         mutex_unlock(&dev->struct_mutex);
2141
2142         ret = __wait_seqno(ring, seqno, true, timeout);
2143         if (timeout) {
2144                 WARN_ON(!timespec_valid(timeout));
2145                 args->timeout_ns = timespec_to_ns(timeout);
2146         }
2147         return ret;
2148
2149 out:
2150         drm_gem_object_unreference(&obj->base);
2151         mutex_unlock(&dev->struct_mutex);
2152         return ret;
2153 }
2154
2155 /**
2156  * i915_gem_object_sync - sync an object to a ring.
2157  *
2158  * @obj: object which may be in use on another ring.
2159  * @to: ring we wish to use the object on. May be NULL.
2160  *
2161  * This code is meant to abstract object synchronization with the GPU.
2162  * Calling with NULL implies synchronizing the object with the CPU
2163  * rather than a particular GPU ring.
2164  *
2165  * Returns 0 if successful, else propagates up the lower layer error.
2166  */
2167 int
2168 i915_gem_object_sync(struct drm_i915_gem_object *obj,
2169                      struct intel_ring_buffer *to)
2170 {
2171         struct intel_ring_buffer *from = obj->ring;
2172         u32 seqno;
2173         int ret, idx;
2174
2175         if (from == NULL || to == from)
2176                 return 0;
2177
2178         if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2179                 return i915_gem_object_wait_rendering(obj, false);
2180
2181         idx = intel_ring_sync_index(from, to);
2182
2183         seqno = obj->last_read_seqno;
2184         if (seqno <= from->sync_seqno[idx])
2185                 return 0;
2186
2187         ret = i915_gem_check_olr(obj->ring, seqno);
2188         if (ret)
2189                 return ret;
2190
2191         ret = to->sync_to(to, from, seqno);
2192         if (!ret)
2193                 from->sync_seqno[idx] = seqno;
2194
2195         return ret;
2196 }
2197
2198 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2199 {
2200         u32 old_write_domain, old_read_domains;
2201
2202         /* Act a barrier for all accesses through the GTT */
2203         mb();
2204
2205         /* Force a pagefault for domain tracking on next user access */
2206         i915_gem_release_mmap(obj);
2207
2208         if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2209                 return;
2210
2211         old_read_domains = obj->base.read_domains;
2212         old_write_domain = obj->base.write_domain;
2213
2214         obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2215         obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2216
2217         trace_i915_gem_object_change_domain(obj,
2218                                             old_read_domains,
2219                                             old_write_domain);
2220 }
2221
2222 /**
2223  * Unbinds an object from the GTT aperture.
2224  */
2225 int
2226 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2227 {
2228         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2229         int ret = 0;
2230
2231         if (obj->gtt_space == NULL)
2232                 return 0;
2233
2234         if (obj->pin_count)
2235                 return -EBUSY;
2236
2237         ret = i915_gem_object_finish_gpu(obj);
2238         if (ret)
2239                 return ret;
2240         /* Continue on if we fail due to EIO, the GPU is hung so we
2241          * should be safe and we need to cleanup or else we might
2242          * cause memory corruption through use-after-free.
2243          */
2244
2245         i915_gem_object_finish_gtt(obj);
2246
2247         /* Move the object to the CPU domain to ensure that
2248          * any possible CPU writes while it's not in the GTT
2249          * are flushed when we go to remap it.
2250          */
2251         if (ret == 0)
2252                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2253         if (ret == -ERESTARTSYS)
2254                 return ret;
2255         if (ret) {
2256                 /* In the event of a disaster, abandon all caches and
2257                  * hope for the best.
2258                  */
2259                 i915_gem_clflush_object(obj);
2260                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2261         }
2262
2263         /* release the fence reg _after_ flushing */
2264         ret = i915_gem_object_put_fence(obj);
2265         if (ret)
2266                 return ret;
2267
2268         trace_i915_gem_object_unbind(obj);
2269
2270         if (obj->has_global_gtt_mapping)
2271                 i915_gem_gtt_unbind_object(obj);
2272         if (obj->has_aliasing_ppgtt_mapping) {
2273                 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2274                 obj->has_aliasing_ppgtt_mapping = 0;
2275         }
2276         i915_gem_gtt_finish_object(obj);
2277
2278         i915_gem_object_put_pages_gtt(obj);
2279
2280         list_del_init(&obj->gtt_list);
2281         list_del_init(&obj->mm_list);
2282         /* Avoid an unnecessary call to unbind on rebind. */
2283         obj->map_and_fenceable = true;
2284
2285         drm_mm_put_block(obj->gtt_space);
2286         obj->gtt_space = NULL;
2287         obj->gtt_offset = 0;
2288
2289         if (i915_gem_object_is_purgeable(obj))
2290                 i915_gem_object_truncate(obj);
2291
2292         return ret;
2293 }
2294
2295 int
2296 i915_gem_flush_ring(struct intel_ring_buffer *ring,
2297                     uint32_t invalidate_domains,
2298                     uint32_t flush_domains)
2299 {
2300         int ret;
2301
2302         if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2303                 return 0;
2304
2305         trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2306
2307         ret = ring->flush(ring, invalidate_domains, flush_domains);
2308         if (ret)
2309                 return ret;
2310
2311         if (flush_domains & I915_GEM_GPU_DOMAINS)
2312                 i915_gem_process_flushing_list(ring, flush_domains);
2313
2314         return 0;
2315 }
2316
2317 static int i915_ring_idle(struct intel_ring_buffer *ring)
2318 {
2319         int ret;
2320
2321         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2322                 return 0;
2323
2324         if (!list_empty(&ring->gpu_write_list)) {
2325                 ret = i915_gem_flush_ring(ring,
2326                                     I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2327                 if (ret)
2328                         return ret;
2329         }
2330
2331         return i915_wait_seqno(ring, i915_gem_next_request_seqno(ring));
2332 }
2333
2334 int i915_gpu_idle(struct drm_device *dev)
2335 {
2336         drm_i915_private_t *dev_priv = dev->dev_private;
2337         struct intel_ring_buffer *ring;
2338         int ret, i;
2339
2340         /* Flush everything onto the inactive list. */
2341         for_each_ring(ring, dev_priv, i) {
2342                 ret = i915_ring_idle(ring);
2343                 if (ret)
2344                         return ret;
2345
2346                 /* Is the device fubar? */
2347                 if (WARN_ON(!list_empty(&ring->gpu_write_list)))
2348                         return -EBUSY;
2349
2350                 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2351                 if (ret)
2352                         return ret;
2353         }
2354
2355         return 0;
2356 }
2357
2358 static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
2359                                         struct drm_i915_gem_object *obj)
2360 {
2361         drm_i915_private_t *dev_priv = dev->dev_private;
2362         uint64_t val;
2363
2364         if (obj) {
2365                 u32 size = obj->gtt_space->size;
2366
2367                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2368                                  0xfffff000) << 32;
2369                 val |= obj->gtt_offset & 0xfffff000;
2370                 val |= (uint64_t)((obj->stride / 128) - 1) <<
2371                         SANDYBRIDGE_FENCE_PITCH_SHIFT;
2372
2373                 if (obj->tiling_mode == I915_TILING_Y)
2374                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2375                 val |= I965_FENCE_REG_VALID;
2376         } else
2377                 val = 0;
2378
2379         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
2380         POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
2381 }
2382
2383 static void i965_write_fence_reg(struct drm_device *dev, int reg,
2384                                  struct drm_i915_gem_object *obj)
2385 {
2386         drm_i915_private_t *dev_priv = dev->dev_private;
2387         uint64_t val;
2388
2389         if (obj) {
2390                 u32 size = obj->gtt_space->size;
2391
2392                 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2393                                  0xfffff000) << 32;
2394                 val |= obj->gtt_offset & 0xfffff000;
2395                 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2396                 if (obj->tiling_mode == I915_TILING_Y)
2397                         val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2398                 val |= I965_FENCE_REG_VALID;
2399         } else
2400                 val = 0;
2401
2402         I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
2403         POSTING_READ(FENCE_REG_965_0 + reg * 8);
2404 }
2405
2406 static void i915_write_fence_reg(struct drm_device *dev, int reg,
2407                                  struct drm_i915_gem_object *obj)
2408 {
2409         drm_i915_private_t *dev_priv = dev->dev_private;
2410         u32 val;
2411
2412         if (obj) {
2413                 u32 size = obj->gtt_space->size;
2414                 int pitch_val;
2415                 int tile_width;
2416
2417                 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2418                      (size & -size) != size ||
2419                      (obj->gtt_offset & (size - 1)),
2420                      "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2421                      obj->gtt_offset, obj->map_and_fenceable, size);
2422
2423                 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2424                         tile_width = 128;
2425                 else
2426                         tile_width = 512;
2427
2428                 /* Note: pitch better be a power of two tile widths */
2429                 pitch_val = obj->stride / tile_width;
2430                 pitch_val = ffs(pitch_val) - 1;
2431
2432                 val = obj->gtt_offset;
2433                 if (obj->tiling_mode == I915_TILING_Y)
2434                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2435                 val |= I915_FENCE_SIZE_BITS(size);
2436                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2437                 val |= I830_FENCE_REG_VALID;
2438         } else
2439                 val = 0;
2440
2441         if (reg < 8)
2442                 reg = FENCE_REG_830_0 + reg * 4;
2443         else
2444                 reg = FENCE_REG_945_8 + (reg - 8) * 4;
2445
2446         I915_WRITE(reg, val);
2447         POSTING_READ(reg);
2448 }
2449
2450 static void i830_write_fence_reg(struct drm_device *dev, int reg,
2451                                 struct drm_i915_gem_object *obj)
2452 {
2453         drm_i915_private_t *dev_priv = dev->dev_private;
2454         uint32_t val;
2455
2456         if (obj) {
2457                 u32 size = obj->gtt_space->size;
2458                 uint32_t pitch_val;
2459
2460                 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2461                      (size & -size) != size ||
2462                      (obj->gtt_offset & (size - 1)),
2463                      "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2464                      obj->gtt_offset, size);
2465
2466                 pitch_val = obj->stride / 128;
2467                 pitch_val = ffs(pitch_val) - 1;
2468
2469                 val = obj->gtt_offset;
2470                 if (obj->tiling_mode == I915_TILING_Y)
2471                         val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2472                 val |= I830_FENCE_SIZE_BITS(size);
2473                 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2474                 val |= I830_FENCE_REG_VALID;
2475         } else
2476                 val = 0;
2477
2478         I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2479         POSTING_READ(FENCE_REG_830_0 + reg * 4);
2480 }
2481
2482 static void i915_gem_write_fence(struct drm_device *dev, int reg,
2483                                  struct drm_i915_gem_object *obj)
2484 {
2485         switch (INTEL_INFO(dev)->gen) {
2486         case 7:
2487         case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
2488         case 5:
2489         case 4: i965_write_fence_reg(dev, reg, obj); break;
2490         case 3: i915_write_fence_reg(dev, reg, obj); break;
2491         case 2: i830_write_fence_reg(dev, reg, obj); break;
2492         default: break;
2493         }
2494 }
2495
2496 static inline int fence_number(struct drm_i915_private *dev_priv,
2497                                struct drm_i915_fence_reg *fence)
2498 {
2499         return fence - dev_priv->fence_regs;
2500 }
2501
2502 static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2503                                          struct drm_i915_fence_reg *fence,
2504                                          bool enable)
2505 {
2506         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2507         int reg = fence_number(dev_priv, fence);
2508
2509         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
2510
2511         if (enable) {
2512                 obj->fence_reg = reg;
2513                 fence->obj = obj;
2514                 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2515         } else {
2516                 obj->fence_reg = I915_FENCE_REG_NONE;
2517                 fence->obj = NULL;
2518                 list_del_init(&fence->lru_list);
2519         }
2520 }
2521
2522 static int
2523 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
2524 {
2525         int ret;
2526
2527         if (obj->fenced_gpu_access) {
2528                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2529                         ret = i915_gem_flush_ring(obj->ring,
2530                                                   0, obj->base.write_domain);
2531                         if (ret)
2532                                 return ret;
2533                 }
2534
2535                 obj->fenced_gpu_access = false;
2536         }
2537
2538         if (obj->last_fenced_seqno) {
2539                 ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
2540                 if (ret)
2541                         return ret;
2542
2543                 obj->last_fenced_seqno = 0;
2544         }
2545
2546         /* Ensure that all CPU reads are completed before installing a fence
2547          * and all writes before removing the fence.
2548          */
2549         if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2550                 mb();
2551
2552         return 0;
2553 }
2554
2555 int
2556 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2557 {
2558         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2559         int ret;
2560
2561         ret = i915_gem_object_flush_fence(obj);
2562         if (ret)
2563                 return ret;
2564
2565         if (obj->fence_reg == I915_FENCE_REG_NONE)
2566                 return 0;
2567
2568         i915_gem_object_update_fence(obj,
2569                                      &dev_priv->fence_regs[obj->fence_reg],
2570                                      false);
2571         i915_gem_object_fence_lost(obj);
2572
2573         return 0;
2574 }
2575
2576 static struct drm_i915_fence_reg *
2577 i915_find_fence_reg(struct drm_device *dev)
2578 {
2579         struct drm_i915_private *dev_priv = dev->dev_private;
2580         struct drm_i915_fence_reg *reg, *avail;
2581         int i;
2582
2583         /* First try to find a free reg */
2584         avail = NULL;
2585         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2586                 reg = &dev_priv->fence_regs[i];
2587                 if (!reg->obj)
2588                         return reg;
2589
2590                 if (!reg->pin_count)
2591                         avail = reg;
2592         }
2593
2594         if (avail == NULL)
2595                 return NULL;
2596
2597         /* None available, try to steal one or wait for a user to finish */
2598         list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2599                 if (reg->pin_count)
2600                         continue;
2601
2602                 return reg;
2603         }
2604
2605         return NULL;
2606 }
2607
2608 /**
2609  * i915_gem_object_get_fence - set up fencing for an object
2610  * @obj: object to map through a fence reg
2611  *
2612  * When mapping objects through the GTT, userspace wants to be able to write
2613  * to them without having to worry about swizzling if the object is tiled.
2614  * This function walks the fence regs looking for a free one for @obj,
2615  * stealing one if it can't find any.
2616  *
2617  * It then sets up the reg based on the object's properties: address, pitch
2618  * and tiling format.
2619  *
2620  * For an untiled surface, this removes any existing fence.
2621  */
2622 int
2623 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
2624 {
2625         struct drm_device *dev = obj->base.dev;
2626         struct drm_i915_private *dev_priv = dev->dev_private;
2627         bool enable = obj->tiling_mode != I915_TILING_NONE;
2628         struct drm_i915_fence_reg *reg;
2629         int ret;
2630
2631         /* Have we updated the tiling parameters upon the object and so
2632          * will need to serialise the write to the associated fence register?
2633          */
2634         if (obj->fence_dirty) {
2635                 ret = i915_gem_object_flush_fence(obj);
2636                 if (ret)
2637                         return ret;
2638         }
2639
2640         /* Just update our place in the LRU if our fence is getting reused. */
2641         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2642                 reg = &dev_priv->fence_regs[obj->fence_reg];
2643                 if (!obj->fence_dirty) {
2644                         list_move_tail(&reg->lru_list,
2645                                        &dev_priv->mm.fence_list);
2646                         return 0;
2647                 }
2648         } else if (enable) {
2649                 reg = i915_find_fence_reg(dev);
2650                 if (reg == NULL)
2651                         return -EDEADLK;
2652
2653                 if (reg->obj) {
2654                         struct drm_i915_gem_object *old = reg->obj;
2655
2656                         ret = i915_gem_object_flush_fence(old);
2657                         if (ret)
2658                                 return ret;
2659
2660                         i915_gem_object_fence_lost(old);
2661                 }
2662         } else
2663                 return 0;
2664
2665         i915_gem_object_update_fence(obj, reg, enable);
2666         obj->fence_dirty = false;
2667
2668         return 0;
2669 }
2670
2671 /**
2672  * Finds free space in the GTT aperture and binds the object there.
2673  */
2674 static int
2675 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2676                             unsigned alignment,
2677                             bool map_and_fenceable)
2678 {
2679         struct drm_device *dev = obj->base.dev;
2680         drm_i915_private_t *dev_priv = dev->dev_private;
2681         struct drm_mm_node *free_space;
2682         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2683         u32 size, fence_size, fence_alignment, unfenced_alignment;
2684         bool mappable, fenceable;
2685         int ret;
2686
2687         if (obj->madv != I915_MADV_WILLNEED) {
2688                 DRM_ERROR("Attempting to bind a purgeable object\n");
2689                 return -EINVAL;
2690         }
2691
2692         fence_size = i915_gem_get_gtt_size(dev,
2693                                            obj->base.size,
2694                                            obj->tiling_mode);
2695         fence_alignment = i915_gem_get_gtt_alignment(dev,
2696                                                      obj->base.size,
2697                                                      obj->tiling_mode);
2698         unfenced_alignment =
2699                 i915_gem_get_unfenced_gtt_alignment(dev,
2700                                                     obj->base.size,
2701                                                     obj->tiling_mode);
2702
2703         if (alignment == 0)
2704                 alignment = map_and_fenceable ? fence_alignment :
2705                                                 unfenced_alignment;
2706         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2707                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2708                 return -EINVAL;
2709         }
2710
2711         size = map_and_fenceable ? fence_size : obj->base.size;
2712
2713         /* If the object is bigger than the entire aperture, reject it early
2714          * before evicting everything in a vain attempt to find space.
2715          */
2716         if (obj->base.size >
2717             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2718                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2719                 return -E2BIG;
2720         }
2721
2722  search_free:
2723         if (map_and_fenceable)
2724                 free_space =
2725                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2726                                                     size, alignment,
2727                                                     0, dev_priv->mm.gtt_mappable_end,
2728                                                     0);
2729         else
2730                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2731                                                 size, alignment, 0);
2732
2733         if (free_space != NULL) {
2734                 if (map_and_fenceable)
2735                         obj->gtt_space =
2736                                 drm_mm_get_block_range_generic(free_space,
2737                                                                size, alignment, 0,
2738                                                                0, dev_priv->mm.gtt_mappable_end,
2739                                                                0);
2740                 else
2741                         obj->gtt_space =
2742                                 drm_mm_get_block(free_space, size, alignment);
2743         }
2744         if (obj->gtt_space == NULL) {
2745                 /* If the gtt is empty and we're still having trouble
2746                  * fitting our object in, we're out of memory.
2747                  */
2748                 ret = i915_gem_evict_something(dev, size, alignment,
2749                                                map_and_fenceable);
2750                 if (ret)
2751                         return ret;
2752
2753                 goto search_free;
2754         }
2755
2756         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2757         if (ret) {
2758                 drm_mm_put_block(obj->gtt_space);
2759                 obj->gtt_space = NULL;
2760
2761                 if (ret == -ENOMEM) {
2762                         /* first try to reclaim some memory by clearing the GTT */
2763                         ret = i915_gem_evict_everything(dev, false);
2764                         if (ret) {
2765                                 /* now try to shrink everyone else */
2766                                 if (gfpmask) {
2767                                         gfpmask = 0;
2768                                         goto search_free;
2769                                 }
2770
2771                                 return -ENOMEM;
2772                         }
2773
2774                         goto search_free;
2775                 }
2776
2777                 return ret;
2778         }
2779
2780         ret = i915_gem_gtt_prepare_object(obj);
2781         if (ret) {
2782                 i915_gem_object_put_pages_gtt(obj);
2783                 drm_mm_put_block(obj->gtt_space);
2784                 obj->gtt_space = NULL;
2785
2786                 if (i915_gem_evict_everything(dev, false))
2787                         return ret;
2788
2789                 goto search_free;
2790         }
2791
2792         if (!dev_priv->mm.aliasing_ppgtt)
2793                 i915_gem_gtt_bind_object(obj, obj->cache_level);
2794
2795         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2796         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2797
2798         /* Assert that the object is not currently in any GPU domain. As it
2799          * wasn't in the GTT, there shouldn't be any way it could have been in
2800          * a GPU cache
2801          */
2802         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2803         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2804
2805         obj->gtt_offset = obj->gtt_space->start;
2806
2807         fenceable =
2808                 obj->gtt_space->size == fence_size &&
2809                 (obj->gtt_space->start & (fence_alignment - 1)) == 0;
2810
2811         mappable =
2812                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2813
2814         obj->map_and_fenceable = mappable && fenceable;
2815
2816         trace_i915_gem_object_bind(obj, map_and_fenceable);
2817         return 0;
2818 }
2819
2820 void
2821 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2822 {
2823         /* If we don't have a page list set up, then we're not pinned
2824          * to GPU, and we can ignore the cache flush because it'll happen
2825          * again at bind time.
2826          */
2827         if (obj->pages == NULL)
2828                 return;
2829
2830         /* If the GPU is snooping the contents of the CPU cache,
2831          * we do not need to manually clear the CPU cache lines.  However,
2832          * the caches are only snooped when the render cache is
2833          * flushed/invalidated.  As we always have to emit invalidations
2834          * and flushes when moving into and out of the RENDER domain, correct
2835          * snooping behaviour occurs naturally as the result of our domain
2836          * tracking.
2837          */
2838         if (obj->cache_level != I915_CACHE_NONE)
2839                 return;
2840
2841         trace_i915_gem_object_clflush(obj);
2842
2843         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2844 }
2845
2846 /** Flushes any GPU write domain for the object if it's dirty. */
2847 static int
2848 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2849 {
2850         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2851                 return 0;
2852
2853         /* Queue the GPU write cache flushing we need. */
2854         return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2855 }
2856
2857 /** Flushes the GTT write domain for the object if it's dirty. */
2858 static void
2859 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2860 {
2861         uint32_t old_write_domain;
2862
2863         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2864                 return;
2865
2866         /* No actual flushing is required for the GTT write domain.  Writes
2867          * to it immediately go to main memory as far as we know, so there's
2868          * no chipset flush.  It also doesn't land in render cache.
2869          *
2870          * However, we do have to enforce the order so that all writes through
2871          * the GTT land before any writes to the device, such as updates to
2872          * the GATT itself.
2873          */
2874         wmb();
2875
2876         old_write_domain = obj->base.write_domain;
2877         obj->base.write_domain = 0;
2878
2879         trace_i915_gem_object_change_domain(obj,
2880                                             obj->base.read_domains,
2881                                             old_write_domain);
2882 }
2883
2884 /** Flushes the CPU write domain for the object if it's dirty. */
2885 static void
2886 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2887 {
2888         uint32_t old_write_domain;
2889
2890         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2891                 return;
2892
2893         i915_gem_clflush_object(obj);
2894         intel_gtt_chipset_flush();
2895         old_write_domain = obj->base.write_domain;
2896         obj->base.write_domain = 0;
2897
2898         trace_i915_gem_object_change_domain(obj,
2899                                             obj->base.read_domains,
2900                                             old_write_domain);
2901 }
2902
2903 /**
2904  * Moves a single object to the GTT read, and possibly write domain.
2905  *
2906  * This function returns when the move is complete, including waiting on
2907  * flushes to occur.
2908  */
2909 int
2910 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2911 {
2912         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
2913         uint32_t old_write_domain, old_read_domains;
2914         int ret;
2915
2916         /* Not valid to be called on unbound objects. */
2917         if (obj->gtt_space == NULL)
2918                 return -EINVAL;
2919
2920         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2921                 return 0;
2922
2923         ret = i915_gem_object_flush_gpu_write_domain(obj);
2924         if (ret)
2925                 return ret;
2926
2927         ret = i915_gem_object_wait_rendering(obj, !write);
2928         if (ret)
2929                 return ret;
2930
2931         i915_gem_object_flush_cpu_write_domain(obj);
2932
2933         old_write_domain = obj->base.write_domain;
2934         old_read_domains = obj->base.read_domains;
2935
2936         /* It should now be out of any other write domains, and we can update
2937          * the domain values for our changes.
2938          */
2939         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2940         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2941         if (write) {
2942                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2943                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2944                 obj->dirty = 1;
2945         }
2946
2947         trace_i915_gem_object_change_domain(obj,
2948                                             old_read_domains,
2949                                             old_write_domain);
2950
2951         /* And bump the LRU for this access */
2952         if (i915_gem_object_is_inactive(obj))
2953                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2954
2955         return 0;
2956 }
2957
2958 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2959                                     enum i915_cache_level cache_level)
2960 {
2961         struct drm_device *dev = obj->base.dev;
2962         drm_i915_private_t *dev_priv = dev->dev_private;
2963         int ret;
2964
2965         if (obj->cache_level == cache_level)
2966                 return 0;
2967
2968         if (obj->pin_count) {
2969                 DRM_DEBUG("can not change the cache level of pinned objects\n");
2970                 return -EBUSY;
2971         }
2972
2973         if (obj->gtt_space) {
2974                 ret = i915_gem_object_finish_gpu(obj);
2975                 if (ret)
2976                         return ret;
2977
2978                 i915_gem_object_finish_gtt(obj);
2979
2980                 /* Before SandyBridge, you could not use tiling or fence
2981                  * registers with snooped memory, so relinquish any fences
2982                  * currently pointing to our region in the aperture.
2983                  */
2984                 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2985                         ret = i915_gem_object_put_fence(obj);
2986                         if (ret)
2987                                 return ret;
2988                 }
2989
2990                 if (obj->has_global_gtt_mapping)
2991                         i915_gem_gtt_bind_object(obj, cache_level);
2992                 if (obj->has_aliasing_ppgtt_mapping)
2993                         i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2994                                                obj, cache_level);
2995         }
2996
2997         if (cache_level == I915_CACHE_NONE) {
2998                 u32 old_read_domains, old_write_domain;
2999
3000                 /* If we're coming from LLC cached, then we haven't
3001                  * actually been tracking whether the data is in the
3002                  * CPU cache or not, since we only allow one bit set
3003                  * in obj->write_domain and have been skipping the clflushes.
3004                  * Just set it to the CPU cache for now.
3005                  */
3006                 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3007                 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
3008
3009                 old_read_domains = obj->base.read_domains;
3010                 old_write_domain = obj->base.write_domain;
3011
3012                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3013                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3014
3015                 trace_i915_gem_object_change_domain(obj,
3016                                                     old_read_domains,
3017                                                     old_write_domain);
3018         }
3019
3020         obj->cache_level = cache_level;
3021         return 0;
3022 }
3023
3024 /*
3025  * Prepare buffer for display plane (scanout, cursors, etc).
3026  * Can be called from an uninterruptible phase (modesetting) and allows
3027  * any flushes to be pipelined (for pageflips).
3028  */
3029 int
3030 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3031                                      u32 alignment,
3032                                      struct intel_ring_buffer *pipelined)
3033 {
3034         u32 old_read_domains, old_write_domain;
3035         int ret;
3036
3037         ret = i915_gem_object_flush_gpu_write_domain(obj);
3038         if (ret)
3039                 return ret;
3040
3041         if (pipelined != obj->ring) {
3042                 ret = i915_gem_object_sync(obj, pipelined);
3043                 if (ret)
3044                         return ret;
3045         }
3046
3047         /* The display engine is not coherent with the LLC cache on gen6.  As
3048          * a result, we make sure that the pinning that is about to occur is
3049          * done with uncached PTEs. This is lowest common denominator for all
3050          * chipsets.
3051          *
3052          * However for gen6+, we could do better by using the GFDT bit instead
3053          * of uncaching, which would allow us to flush all the LLC-cached data
3054          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3055          */
3056         ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
3057         if (ret)
3058                 return ret;
3059
3060         /* As the user may map the buffer once pinned in the display plane
3061          * (e.g. libkms for the bootup splash), we have to ensure that we
3062          * always use map_and_fenceable for all scanout buffers.
3063          */
3064         ret = i915_gem_object_pin(obj, alignment, true);
3065         if (ret)
3066                 return ret;
3067
3068         i915_gem_object_flush_cpu_write_domain(obj);
3069
3070         old_write_domain = obj->base.write_domain;
3071         old_read_domains = obj->base.read_domains;
3072
3073         /* It should now be out of any other write domains, and we can update
3074          * the domain values for our changes.
3075          */
3076         obj->base.write_domain = 0;
3077         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3078
3079         trace_i915_gem_object_change_domain(obj,
3080                                             old_read_domains,
3081                                             old_write_domain);
3082
3083         return 0;
3084 }
3085
3086 int
3087 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3088 {
3089         int ret;
3090
3091         if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3092                 return 0;
3093
3094         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3095                 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
3096                 if (ret)
3097                         return ret;
3098         }
3099
3100         ret = i915_gem_object_wait_rendering(obj, false);
3101         if (ret)
3102                 return ret;
3103
3104         /* Ensure that we invalidate the GPU's caches and TLBs. */
3105         obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3106         return 0;
3107 }
3108
3109 /**
3110  * Moves a single object to the CPU read, and possibly write domain.
3111  *
3112  * This function returns when the move is complete, including waiting on
3113  * flushes to occur.
3114  */
3115 int
3116 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3117 {
3118         uint32_t old_write_domain, old_read_domains;
3119         int ret;
3120
3121         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3122                 return 0;
3123
3124         ret = i915_gem_object_flush_gpu_write_domain(obj);
3125         if (ret)
3126                 return ret;
3127
3128         ret = i915_gem_object_wait_rendering(obj, !write);
3129         if (ret)
3130                 return ret;
3131
3132         i915_gem_object_flush_gtt_write_domain(obj);
3133
3134         old_write_domain = obj->base.write_domain;
3135         old_read_domains = obj->base.read_domains;
3136
3137         /* Flush the CPU cache if it's still invalid. */
3138         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3139                 i915_gem_clflush_object(obj);
3140
3141                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3142         }
3143
3144         /* It should now be out of any other write domains, and we can update
3145          * the domain values for our changes.
3146          */
3147         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3148
3149         /* If we're writing through the CPU, then the GPU read domains will
3150          * need to be invalidated at next use.
3151          */
3152         if (write) {
3153                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3154                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3155         }
3156
3157         trace_i915_gem_object_change_domain(obj,
3158                                             old_read_domains,
3159                                             old_write_domain);
3160
3161         return 0;
3162 }
3163
3164 /* Throttle our rendering by waiting until the ring has completed our requests
3165  * emitted over 20 msec ago.
3166  *
3167  * Note that if we were to use the current jiffies each time around the loop,
3168  * we wouldn't escape the function with any frames outstanding if the time to
3169  * render a frame was over 20ms.
3170  *
3171  * This should get us reasonable parallelism between CPU and GPU but also
3172  * relatively low latency when blocking on a particular request to finish.
3173  */
3174 static int
3175 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3176 {
3177         struct drm_i915_private *dev_priv = dev->dev_private;
3178         struct drm_i915_file_private *file_priv = file->driver_priv;
3179         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3180         struct drm_i915_gem_request *request;
3181         struct intel_ring_buffer *ring = NULL;
3182         u32 seqno = 0;
3183         int ret;
3184
3185         if (atomic_read(&dev_priv->mm.wedged))
3186                 return -EIO;
3187
3188         spin_lock(&file_priv->mm.lock);
3189         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3190                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3191                         break;
3192
3193                 ring = request->ring;
3194                 seqno = request->seqno;
3195         }
3196         spin_unlock(&file_priv->mm.lock);
3197
3198         if (seqno == 0)
3199                 return 0;
3200
3201         ret = __wait_seqno(ring, seqno, true, NULL);
3202         if (ret == 0)
3203                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3204
3205         return ret;
3206 }
3207
3208 int
3209 i915_gem_object_pin(struct drm_i915_gem_object *obj,
3210                     uint32_t alignment,
3211                     bool map_and_fenceable)
3212 {
3213         int ret;
3214
3215         BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
3216
3217         if (obj->gtt_space != NULL) {
3218                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3219                     (map_and_fenceable && !obj->map_and_fenceable)) {
3220                         WARN(obj->pin_count,
3221                              "bo is already pinned with incorrect alignment:"
3222                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3223                              " obj->map_and_fenceable=%d\n",
3224                              obj->gtt_offset, alignment,
3225                              map_and_fenceable,
3226                              obj->map_and_fenceable);
3227                         ret = i915_gem_object_unbind(obj);
3228                         if (ret)
3229                                 return ret;
3230                 }
3231         }
3232
3233         if (obj->gtt_space == NULL) {
3234                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3235                                                   map_and_fenceable);
3236                 if (ret)
3237                         return ret;
3238         }
3239
3240         if (!obj->has_global_gtt_mapping && map_and_fenceable)
3241                 i915_gem_gtt_bind_object(obj, obj->cache_level);
3242
3243         obj->pin_count++;
3244         obj->pin_mappable |= map_and_fenceable;
3245
3246         return 0;
3247 }
3248
3249 void
3250 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3251 {
3252         BUG_ON(obj->pin_count == 0);
3253         BUG_ON(obj->gtt_space == NULL);
3254
3255         if (--obj->pin_count == 0)
3256                 obj->pin_mappable = false;
3257 }
3258
3259 int
3260 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3261                    struct drm_file *file)
3262 {
3263         struct drm_i915_gem_pin *args = data;
3264         struct drm_i915_gem_object *obj;
3265         int ret;
3266
3267         ret = i915_mutex_lock_interruptible(dev);
3268         if (ret)
3269                 return ret;
3270
3271         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3272         if (&obj->base == NULL) {
3273                 ret = -ENOENT;
3274                 goto unlock;
3275         }
3276
3277         if (obj->madv != I915_MADV_WILLNEED) {
3278                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3279                 ret = -EINVAL;
3280                 goto out;
3281         }
3282
3283         if (obj->pin_filp != NULL && obj->pin_filp != file) {
3284                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3285                           args->handle);
3286                 ret = -EINVAL;
3287                 goto out;
3288         }
3289
3290         obj->user_pin_count++;
3291         obj->pin_filp = file;
3292         if (obj->user_pin_count == 1) {
3293                 ret = i915_gem_object_pin(obj, args->alignment, true);
3294                 if (ret)
3295                         goto out;
3296         }
3297
3298         /* XXX - flush the CPU caches for pinned objects
3299          * as the X server doesn't manage domains yet
3300          */
3301         i915_gem_object_flush_cpu_write_domain(obj);
3302         args->offset = obj->gtt_offset;
3303 out:
3304         drm_gem_object_unreference(&obj->base);
3305 unlock:
3306         mutex_unlock(&dev->struct_mutex);
3307         return ret;
3308 }
3309
3310 int
3311 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3312                      struct drm_file *file)
3313 {
3314         struct drm_i915_gem_pin *args = data;
3315         struct drm_i915_gem_object *obj;
3316         int ret;
3317
3318         ret = i915_mutex_lock_interruptible(dev);
3319         if (ret)
3320                 return ret;
3321
3322         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3323         if (&obj->base == NULL) {
3324                 ret = -ENOENT;
3325                 goto unlock;
3326         }
3327
3328         if (obj->pin_filp != file) {
3329                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3330                           args->handle);
3331                 ret = -EINVAL;
3332                 goto out;
3333         }
3334         obj->user_pin_count--;
3335         if (obj->user_pin_count == 0) {
3336                 obj->pin_filp = NULL;
3337                 i915_gem_object_unpin(obj);
3338         }
3339
3340 out:
3341         drm_gem_object_unreference(&obj->base);
3342 unlock:
3343         mutex_unlock(&dev->struct_mutex);
3344         return ret;
3345 }
3346
3347 int
3348 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3349                     struct drm_file *file)
3350 {
3351         struct drm_i915_gem_busy *args = data;
3352         struct drm_i915_gem_object *obj;
3353         int ret;
3354
3355         ret = i915_mutex_lock_interruptible(dev);
3356         if (ret)
3357                 return ret;
3358
3359         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3360         if (&obj->base == NULL) {
3361                 ret = -ENOENT;
3362                 goto unlock;
3363         }
3364
3365         /* Count all active objects as busy, even if they are currently not used
3366          * by the gpu. Users of this interface expect objects to eventually
3367          * become non-busy without any further actions, therefore emit any
3368          * necessary flushes here.
3369          */
3370         ret = i915_gem_object_flush_active(obj);
3371
3372         args->busy = obj->active;
3373         if (obj->ring) {
3374                 BUILD_BUG_ON(I915_NUM_RINGS > 16);
3375                 args->busy |= intel_ring_flag(obj->ring) << 16;
3376         }
3377
3378         drm_gem_object_unreference(&obj->base);
3379 unlock:
3380         mutex_unlock(&dev->struct_mutex);
3381         return ret;
3382 }
3383
3384 int
3385 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3386                         struct drm_file *file_priv)
3387 {
3388         return i915_gem_ring_throttle(dev, file_priv);
3389 }
3390
3391 int
3392 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3393                        struct drm_file *file_priv)
3394 {
3395         struct drm_i915_gem_madvise *args = data;
3396         struct drm_i915_gem_object *obj;
3397         int ret;
3398
3399         switch (args->madv) {
3400         case I915_MADV_DONTNEED:
3401         case I915_MADV_WILLNEED:
3402             break;
3403         default:
3404             return -EINVAL;
3405         }
3406
3407         ret = i915_mutex_lock_interruptible(dev);
3408         if (ret)
3409                 return ret;
3410
3411         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3412         if (&obj->base == NULL) {
3413                 ret = -ENOENT;
3414                 goto unlock;
3415         }
3416
3417         if (obj->pin_count) {
3418                 ret = -EINVAL;
3419                 goto out;
3420         }
3421
3422         if (obj->madv != __I915_MADV_PURGED)
3423                 obj->madv = args->madv;
3424
3425         /* if the object is no longer bound, discard its backing storage */
3426         if (i915_gem_object_is_purgeable(obj) &&
3427             obj->gtt_space == NULL)
3428                 i915_gem_object_truncate(obj);
3429
3430         args->retained = obj->madv != __I915_MADV_PURGED;
3431
3432 out:
3433         drm_gem_object_unreference(&obj->base);
3434 unlock:
3435         mutex_unlock(&dev->struct_mutex);
3436         return ret;
3437 }
3438
3439 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3440                                                   size_t size)
3441 {
3442         struct drm_i915_private *dev_priv = dev->dev_private;
3443         struct drm_i915_gem_object *obj;
3444         struct address_space *mapping;
3445         u32 mask;
3446
3447         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3448         if (obj == NULL)
3449                 return NULL;
3450
3451         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3452                 kfree(obj);
3453                 return NULL;
3454         }
3455
3456         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
3457         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
3458                 /* 965gm cannot relocate objects above 4GiB. */
3459                 mask &= ~__GFP_HIGHMEM;
3460                 mask |= __GFP_DMA32;
3461         }
3462
3463         mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3464         mapping_set_gfp_mask(mapping, mask);
3465
3466         i915_gem_info_add_obj(dev_priv, size);
3467
3468         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3469         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3470
3471         if (HAS_LLC(dev)) {
3472                 /* On some devices, we can have the GPU use the LLC (the CPU
3473                  * cache) for about a 10% performance improvement
3474                  * compared to uncached.  Graphics requests other than
3475                  * display scanout are coherent with the CPU in
3476                  * accessing this cache.  This means in this mode we
3477                  * don't need to clflush on the CPU side, and on the
3478                  * GPU side we only need to flush internal caches to
3479                  * get data visible to the CPU.
3480                  *
3481                  * However, we maintain the display planes as UC, and so
3482                  * need to rebind when first used as such.
3483                  */
3484                 obj->cache_level = I915_CACHE_LLC;
3485         } else
3486                 obj->cache_level = I915_CACHE_NONE;
3487
3488         obj->base.driver_private = NULL;
3489         obj->fence_reg = I915_FENCE_REG_NONE;
3490         INIT_LIST_HEAD(&obj->mm_list);
3491         INIT_LIST_HEAD(&obj->gtt_list);
3492         INIT_LIST_HEAD(&obj->ring_list);
3493         INIT_LIST_HEAD(&obj->exec_list);
3494         INIT_LIST_HEAD(&obj->gpu_write_list);
3495         obj->madv = I915_MADV_WILLNEED;
3496         /* Avoid an unnecessary call to unbind on the first bind. */
3497         obj->map_and_fenceable = true;
3498
3499         return obj;
3500 }
3501
3502 int i915_gem_init_object(struct drm_gem_object *obj)
3503 {
3504         BUG();
3505
3506         return 0;
3507 }
3508
3509 void i915_gem_free_object(struct drm_gem_object *gem_obj)
3510 {
3511         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3512         struct drm_device *dev = obj->base.dev;
3513         drm_i915_private_t *dev_priv = dev->dev_private;
3514
3515         trace_i915_gem_object_destroy(obj);
3516
3517         if (gem_obj->import_attach)
3518                 drm_prime_gem_destroy(gem_obj, obj->sg_table);
3519
3520         if (obj->phys_obj)
3521                 i915_gem_detach_phys_object(dev, obj);
3522
3523         obj->pin_count = 0;
3524         if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
3525                 bool was_interruptible;
3526
3527                 was_interruptible = dev_priv->mm.interruptible;
3528                 dev_priv->mm.interruptible = false;
3529
3530                 WARN_ON(i915_gem_object_unbind(obj));
3531
3532                 dev_priv->mm.interruptible = was_interruptible;
3533         }
3534
3535         if (obj->base.map_list.map)
3536                 drm_gem_free_mmap_offset(&obj->base);
3537
3538         drm_gem_object_release(&obj->base);
3539         i915_gem_info_remove_obj(dev_priv, obj->base.size);
3540
3541         kfree(obj->bit_17);
3542         kfree(obj);
3543 }
3544
3545 int
3546 i915_gem_idle(struct drm_device *dev)
3547 {
3548         drm_i915_private_t *dev_priv = dev->dev_private;
3549         int ret;
3550
3551         mutex_lock(&dev->struct_mutex);
3552
3553         if (dev_priv->mm.suspended) {
3554                 mutex_unlock(&dev->struct_mutex);
3555                 return 0;
3556         }
3557
3558         ret = i915_gpu_idle(dev);
3559         if (ret) {
3560                 mutex_unlock(&dev->struct_mutex);
3561                 return ret;
3562         }
3563         i915_gem_retire_requests(dev);
3564
3565         /* Under UMS, be paranoid and evict. */
3566         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3567                 i915_gem_evict_everything(dev, false);
3568
3569         i915_gem_reset_fences(dev);
3570
3571         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
3572          * We need to replace this with a semaphore, or something.
3573          * And not confound mm.suspended!
3574          */
3575         dev_priv->mm.suspended = 1;
3576         del_timer_sync(&dev_priv->hangcheck_timer);
3577
3578         i915_kernel_lost_context(dev);
3579         i915_gem_cleanup_ringbuffer(dev);
3580
3581         mutex_unlock(&dev->struct_mutex);
3582
3583         /* Cancel the retire work handler, which should be idle now. */
3584         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3585
3586         return 0;
3587 }
3588
3589 void i915_gem_l3_remap(struct drm_device *dev)
3590 {
3591         drm_i915_private_t *dev_priv = dev->dev_private;
3592         u32 misccpctl;
3593         int i;
3594
3595         if (!IS_IVYBRIDGE(dev))
3596                 return;
3597
3598         if (!dev_priv->mm.l3_remap_info)
3599                 return;
3600
3601         misccpctl = I915_READ(GEN7_MISCCPCTL);
3602         I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
3603         POSTING_READ(GEN7_MISCCPCTL);
3604
3605         for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
3606                 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
3607                 if (remap && remap != dev_priv->mm.l3_remap_info[i/4])
3608                         DRM_DEBUG("0x%x was already programmed to %x\n",
3609                                   GEN7_L3LOG_BASE + i, remap);
3610                 if (remap && !dev_priv->mm.l3_remap_info[i/4])
3611                         DRM_DEBUG_DRIVER("Clearing remapped register\n");
3612                 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->mm.l3_remap_info[i/4]);
3613         }
3614
3615         /* Make sure all the writes land before disabling dop clock gating */
3616         POSTING_READ(GEN7_L3LOG_BASE);
3617
3618         I915_WRITE(GEN7_MISCCPCTL, misccpctl);
3619 }
3620
3621 void i915_gem_init_swizzling(struct drm_device *dev)
3622 {
3623         drm_i915_private_t *dev_priv = dev->dev_private;
3624
3625         if (INTEL_INFO(dev)->gen < 5 ||
3626             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
3627                 return;
3628
3629         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
3630                                  DISP_TILE_SURFACE_SWIZZLING);
3631
3632         if (IS_GEN5(dev))
3633                 return;
3634
3635         I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
3636         if (IS_GEN6(dev))
3637                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
3638         else
3639                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
3640 }
3641
3642 void i915_gem_init_ppgtt(struct drm_device *dev)
3643 {
3644         drm_i915_private_t *dev_priv = dev->dev_private;
3645         uint32_t pd_offset;
3646         struct intel_ring_buffer *ring;
3647         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
3648         uint32_t __iomem *pd_addr;
3649         uint32_t pd_entry;
3650         int i;
3651
3652         if (!dev_priv->mm.aliasing_ppgtt)
3653                 return;
3654
3655
3656         pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t);
3657         for (i = 0; i < ppgtt->num_pd_entries; i++) {
3658                 dma_addr_t pt_addr;
3659
3660                 if (dev_priv->mm.gtt->needs_dmar)
3661                         pt_addr = ppgtt->pt_dma_addr[i];
3662                 else
3663                         pt_addr = page_to_phys(ppgtt->pt_pages[i]);
3664
3665                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
3666                 pd_entry |= GEN6_PDE_VALID;
3667
3668                 writel(pd_entry, pd_addr + i);
3669         }
3670         readl(pd_addr);
3671
3672         pd_offset = ppgtt->pd_offset;
3673         pd_offset /= 64; /* in cachelines, */
3674         pd_offset <<= 16;
3675
3676         if (INTEL_INFO(dev)->gen == 6) {
3677                 uint32_t ecochk, gab_ctl, ecobits;
3678
3679                 ecobits = I915_READ(GAC_ECO_BITS);
3680                 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
3681
3682                 gab_ctl = I915_READ(GAB_CTL);
3683                 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
3684
3685                 ecochk = I915_READ(GAM_ECOCHK);
3686                 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
3687                                        ECOCHK_PPGTT_CACHE64B);
3688                 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
3689         } else if (INTEL_INFO(dev)->gen >= 7) {
3690                 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
3691                 /* GFX_MODE is per-ring on gen7+ */
3692         }
3693
3694         for_each_ring(ring, dev_priv, i) {
3695                 if (INTEL_INFO(dev)->gen >= 7)
3696                         I915_WRITE(RING_MODE_GEN7(ring),
3697                                    _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
3698
3699                 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
3700                 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
3701         }
3702 }
3703
3704 static bool
3705 intel_enable_blt(struct drm_device *dev)
3706 {
3707         if (!HAS_BLT(dev))
3708                 return false;
3709
3710         /* The blitter was dysfunctional on early prototypes */
3711         if (IS_GEN6(dev) && dev->pdev->revision < 8) {
3712                 DRM_INFO("BLT not supported on this pre-production hardware;"
3713                          " graphics performance will be degraded.\n");
3714                 return false;
3715         }
3716
3717         return true;
3718 }
3719
3720 int
3721 i915_gem_init_hw(struct drm_device *dev)
3722 {
3723         drm_i915_private_t *dev_priv = dev->dev_private;
3724         int ret;
3725
3726         if (!intel_enable_gtt())
3727                 return -EIO;
3728
3729         i915_gem_l3_remap(dev);
3730
3731         i915_gem_init_swizzling(dev);
3732
3733         ret = intel_init_render_ring_buffer(dev);
3734         if (ret)
3735                 return ret;
3736
3737         if (HAS_BSD(dev)) {
3738                 ret = intel_init_bsd_ring_buffer(dev);
3739                 if (ret)
3740                         goto cleanup_render_ring;
3741         }
3742
3743         if (intel_enable_blt(dev)) {
3744                 ret = intel_init_blt_ring_buffer(dev);
3745                 if (ret)
3746                         goto cleanup_bsd_ring;
3747         }
3748
3749         dev_priv->next_seqno = 1;
3750
3751         /*
3752          * XXX: There was some w/a described somewhere suggesting loading
3753          * contexts before PPGTT.
3754          */
3755         i915_gem_context_init(dev);
3756         i915_gem_init_ppgtt(dev);
3757
3758         return 0;
3759
3760 cleanup_bsd_ring:
3761         intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
3762 cleanup_render_ring:
3763         intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3764         return ret;
3765 }
3766
3767 static bool
3768 intel_enable_ppgtt(struct drm_device *dev)
3769 {
3770         if (i915_enable_ppgtt >= 0)
3771                 return i915_enable_ppgtt;
3772
3773 #ifdef CONFIG_INTEL_IOMMU
3774         /* Disable ppgtt on SNB if VT-d is on. */
3775         if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
3776                 return false;
3777 #endif
3778
3779         return true;
3780 }
3781
3782 int i915_gem_init(struct drm_device *dev)
3783 {
3784         struct drm_i915_private *dev_priv = dev->dev_private;
3785         unsigned long gtt_size, mappable_size;
3786         int ret;
3787
3788         gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT;
3789         mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT;
3790
3791         mutex_lock(&dev->struct_mutex);
3792         if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
3793                 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
3794                  * aperture accordingly when using aliasing ppgtt. */
3795                 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
3796
3797                 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
3798
3799                 ret = i915_gem_init_aliasing_ppgtt(dev);
3800                 if (ret) {
3801                         mutex_unlock(&dev->struct_mutex);
3802                         return ret;
3803                 }
3804         } else {
3805                 /* Let GEM Manage all of the aperture.
3806                  *
3807                  * However, leave one page at the end still bound to the scratch
3808                  * page.  There are a number of places where the hardware
3809                  * apparently prefetches past the end of the object, and we've
3810                  * seen multiple hangs with the GPU head pointer stuck in a
3811                  * batchbuffer bound at the last page of the aperture.  One page
3812                  * should be enough to keep any prefetching inside of the
3813                  * aperture.
3814                  */
3815                 i915_gem_init_global_gtt(dev, 0, mappable_size,
3816                                          gtt_size);
3817         }
3818
3819         ret = i915_gem_init_hw(dev);
3820         mutex_unlock(&dev->struct_mutex);
3821         if (ret) {
3822                 i915_gem_cleanup_aliasing_ppgtt(dev);
3823                 return ret;
3824         }
3825
3826         /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
3827         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3828                 dev_priv->dri1.allow_batchbuffer = 1;
3829         return 0;
3830 }
3831
3832 void
3833 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3834 {
3835         drm_i915_private_t *dev_priv = dev->dev_private;
3836         struct intel_ring_buffer *ring;
3837         int i;
3838
3839         for_each_ring(ring, dev_priv, i)
3840                 intel_cleanup_ring_buffer(ring);
3841 }
3842
3843 int
3844 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3845                        struct drm_file *file_priv)
3846 {
3847         drm_i915_private_t *dev_priv = dev->dev_private;
3848         int ret;
3849
3850         if (drm_core_check_feature(dev, DRIVER_MODESET))
3851                 return 0;
3852
3853         if (atomic_read(&dev_priv->mm.wedged)) {
3854                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
3855                 atomic_set(&dev_priv->mm.wedged, 0);
3856         }
3857
3858         mutex_lock(&dev->struct_mutex);
3859         dev_priv->mm.suspended = 0;
3860
3861         ret = i915_gem_init_hw(dev);
3862         if (ret != 0) {
3863                 mutex_unlock(&dev->struct_mutex);
3864                 return ret;
3865         }
3866
3867         BUG_ON(!list_empty(&dev_priv->mm.active_list));
3868         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
3869         mutex_unlock(&dev->struct_mutex);
3870
3871         ret = drm_irq_install(dev);
3872         if (ret)
3873                 goto cleanup_ringbuffer;
3874
3875         return 0;
3876
3877 cleanup_ringbuffer:
3878         mutex_lock(&dev->struct_mutex);
3879         i915_gem_cleanup_ringbuffer(dev);
3880         dev_priv->mm.suspended = 1;
3881         mutex_unlock(&dev->struct_mutex);
3882
3883         return ret;
3884 }
3885
3886 int
3887 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3888                        struct drm_file *file_priv)
3889 {
3890         if (drm_core_check_feature(dev, DRIVER_MODESET))
3891                 return 0;
3892
3893         drm_irq_uninstall(dev);
3894         return i915_gem_idle(dev);
3895 }
3896
3897 void
3898 i915_gem_lastclose(struct drm_device *dev)
3899 {
3900         int ret;
3901
3902         if (drm_core_check_feature(dev, DRIVER_MODESET))
3903                 return;
3904
3905         ret = i915_gem_idle(dev);
3906         if (ret)
3907                 DRM_ERROR("failed to idle hardware: %d\n", ret);
3908 }
3909
3910 static void
3911 init_ring_lists(struct intel_ring_buffer *ring)
3912 {
3913         INIT_LIST_HEAD(&ring->active_list);
3914         INIT_LIST_HEAD(&ring->request_list);
3915         INIT_LIST_HEAD(&ring->gpu_write_list);
3916 }
3917
3918 void
3919 i915_gem_load(struct drm_device *dev)
3920 {
3921         int i;
3922         drm_i915_private_t *dev_priv = dev->dev_private;
3923
3924         INIT_LIST_HEAD(&dev_priv->mm.active_list);
3925         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3926         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3927         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3928         for (i = 0; i < I915_NUM_RINGS; i++)
3929                 init_ring_lists(&dev_priv->ring[i]);
3930         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
3931                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3932         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3933                           i915_gem_retire_work_handler);
3934         init_completion(&dev_priv->error_completion);
3935
3936         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3937         if (IS_GEN3(dev)) {
3938                 I915_WRITE(MI_ARB_STATE,
3939                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
3940         }
3941
3942         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3943
3944         /* Old X drivers will take 0-2 for front, back, depth buffers */
3945         if (!drm_core_check_feature(dev, DRIVER_MODESET))
3946                 dev_priv->fence_reg_start = 3;
3947
3948         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3949                 dev_priv->num_fence_regs = 16;
3950         else
3951                 dev_priv->num_fence_regs = 8;
3952
3953         /* Initialize fence registers to zero */
3954         i915_gem_reset_fences(dev);
3955
3956         i915_gem_detect_bit_6_swizzle(dev);
3957         init_waitqueue_head(&dev_priv->pending_flip_queue);
3958
3959         dev_priv->mm.interruptible = true;
3960
3961         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
3962         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
3963         register_shrinker(&dev_priv->mm.inactive_shrinker);
3964 }
3965
3966 /*
3967  * Create a physically contiguous memory object for this object
3968  * e.g. for cursor + overlay regs
3969  */
3970 static int i915_gem_init_phys_object(struct drm_device *dev,
3971                                      int id, int size, int align)
3972 {
3973         drm_i915_private_t *dev_priv = dev->dev_private;
3974         struct drm_i915_gem_phys_object *phys_obj;
3975         int ret;
3976
3977         if (dev_priv->mm.phys_objs[id - 1] || !size)
3978                 return 0;
3979
3980         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
3981         if (!phys_obj)
3982                 return -ENOMEM;
3983
3984         phys_obj->id = id;
3985
3986         phys_obj->handle = drm_pci_alloc(dev, size, align);
3987         if (!phys_obj->handle) {
3988                 ret = -ENOMEM;
3989                 goto kfree_obj;
3990         }
3991 #ifdef CONFIG_X86
3992         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
3993 #endif
3994
3995         dev_priv->mm.phys_objs[id - 1] = phys_obj;
3996
3997         return 0;
3998 kfree_obj:
3999         kfree(phys_obj);
4000         return ret;
4001 }
4002
4003 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4004 {
4005         drm_i915_private_t *dev_priv = dev->dev_private;
4006         struct drm_i915_gem_phys_object *phys_obj;
4007
4008         if (!dev_priv->mm.phys_objs[id - 1])
4009                 return;
4010
4011         phys_obj = dev_priv->mm.phys_objs[id - 1];
4012         if (phys_obj->cur_obj) {
4013                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4014         }
4015
4016 #ifdef CONFIG_X86
4017         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4018 #endif
4019         drm_pci_free(dev, phys_obj->handle);
4020         kfree(phys_obj);
4021         dev_priv->mm.phys_objs[id - 1] = NULL;
4022 }
4023
4024 void i915_gem_free_all_phys_object(struct drm_device *dev)
4025 {
4026         int i;
4027
4028         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4029                 i915_gem_free_phys_object(dev, i);
4030 }
4031
4032 void i915_gem_detach_phys_object(struct drm_device *dev,
4033                                  struct drm_i915_gem_object *obj)
4034 {
4035         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4036         char *vaddr;
4037         int i;
4038         int page_count;
4039
4040         if (!obj->phys_obj)
4041                 return;
4042         vaddr = obj->phys_obj->handle->vaddr;
4043
4044         page_count = obj->base.size / PAGE_SIZE;
4045         for (i = 0; i < page_count; i++) {
4046                 struct page *page = shmem_read_mapping_page(mapping, i);
4047                 if (!IS_ERR(page)) {
4048                         char *dst = kmap_atomic(page);
4049                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4050                         kunmap_atomic(dst);
4051
4052                         drm_clflush_pages(&page, 1);
4053
4054                         set_page_dirty(page);
4055                         mark_page_accessed(page);
4056                         page_cache_release(page);
4057                 }
4058         }
4059         intel_gtt_chipset_flush();
4060
4061         obj->phys_obj->cur_obj = NULL;
4062         obj->phys_obj = NULL;
4063 }
4064
4065 int
4066 i915_gem_attach_phys_object(struct drm_device *dev,
4067                             struct drm_i915_gem_object *obj,
4068                             int id,
4069                             int align)
4070 {
4071         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4072         drm_i915_private_t *dev_priv = dev->dev_private;
4073         int ret = 0;
4074         int page_count;
4075         int i;
4076
4077         if (id > I915_MAX_PHYS_OBJECT)
4078                 return -EINVAL;
4079
4080         if (obj->phys_obj) {
4081                 if (obj->phys_obj->id == id)
4082                         return 0;
4083                 i915_gem_detach_phys_object(dev, obj);
4084         }
4085
4086         /* create a new object */
4087         if (!dev_priv->mm.phys_objs[id - 1]) {
4088                 ret = i915_gem_init_phys_object(dev, id,
4089                                                 obj->base.size, align);
4090                 if (ret) {
4091                         DRM_ERROR("failed to init phys object %d size: %zu\n",
4092                                   id, obj->base.size);
4093                         return ret;
4094                 }
4095         }
4096
4097         /* bind to the object */
4098         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4099         obj->phys_obj->cur_obj = obj;
4100
4101         page_count = obj->base.size / PAGE_SIZE;
4102
4103         for (i = 0; i < page_count; i++) {
4104                 struct page *page;
4105                 char *dst, *src;
4106
4107                 page = shmem_read_mapping_page(mapping, i);
4108                 if (IS_ERR(page))
4109                         return PTR_ERR(page);
4110
4111                 src = kmap_atomic(page);
4112                 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4113                 memcpy(dst, src, PAGE_SIZE);
4114                 kunmap_atomic(src);
4115
4116                 mark_page_accessed(page);
4117                 page_cache_release(page);
4118         }
4119
4120         return 0;
4121 }
4122
4123 static int
4124 i915_gem_phys_pwrite(struct drm_device *dev,
4125                      struct drm_i915_gem_object *obj,
4126                      struct drm_i915_gem_pwrite *args,
4127                      struct drm_file *file_priv)
4128 {
4129         void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4130         char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
4131
4132         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4133                 unsigned long unwritten;
4134
4135                 /* The physical object once assigned is fixed for the lifetime
4136                  * of the obj, so we can safely drop the lock and continue
4137                  * to access vaddr.
4138                  */
4139                 mutex_unlock(&dev->struct_mutex);
4140                 unwritten = copy_from_user(vaddr, user_data, args->size);
4141                 mutex_lock(&dev->struct_mutex);
4142                 if (unwritten)
4143                         return -EFAULT;
4144         }
4145
4146         intel_gtt_chipset_flush();
4147         return 0;
4148 }
4149
4150 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4151 {
4152         struct drm_i915_file_private *file_priv = file->driver_priv;
4153
4154         /* Clean up our request list when the client is going away, so that
4155          * later retire_requests won't dereference our soon-to-be-gone
4156          * file_priv.
4157          */
4158         spin_lock(&file_priv->mm.lock);
4159         while (!list_empty(&file_priv->mm.request_list)) {
4160                 struct drm_i915_gem_request *request;
4161
4162                 request = list_first_entry(&file_priv->mm.request_list,
4163                                            struct drm_i915_gem_request,
4164                                            client_list);
4165                 list_del(&request->client_list);
4166                 request->file_priv = NULL;
4167         }
4168         spin_unlock(&file_priv->mm.lock);
4169 }
4170
4171 static int
4172 i915_gpu_is_active(struct drm_device *dev)
4173 {
4174         drm_i915_private_t *dev_priv = dev->dev_private;
4175         return !list_empty(&dev_priv->mm.active_list);
4176 }
4177
4178 static int
4179 i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
4180 {
4181         struct drm_i915_private *dev_priv =
4182                 container_of(shrinker,
4183                              struct drm_i915_private,
4184                              mm.inactive_shrinker);
4185         struct drm_device *dev = dev_priv->dev;
4186         struct drm_i915_gem_object *obj, *next;
4187         int nr_to_scan = sc->nr_to_scan;
4188         int cnt;
4189
4190         if (!mutex_trylock(&dev->struct_mutex))
4191                 return 0;
4192
4193         /* "fast-path" to count number of available objects */
4194         if (nr_to_scan == 0) {
4195                 cnt = 0;
4196                 list_for_each_entry(obj,
4197                                     &dev_priv->mm.inactive_list,
4198                                     mm_list)
4199                         cnt++;
4200                 mutex_unlock(&dev->struct_mutex);
4201                 return cnt / 100 * sysctl_vfs_cache_pressure;
4202         }
4203
4204 rescan:
4205         /* first scan for clean buffers */
4206         i915_gem_retire_requests(dev);
4207
4208         list_for_each_entry_safe(obj, next,
4209                                  &dev_priv->mm.inactive_list,
4210                                  mm_list) {
4211                 if (i915_gem_object_is_purgeable(obj)) {
4212                         if (i915_gem_object_unbind(obj) == 0 &&
4213                             --nr_to_scan == 0)
4214                                 break;
4215                 }
4216         }
4217
4218         /* second pass, evict/count anything still on the inactive list */
4219         cnt = 0;
4220         list_for_each_entry_safe(obj, next,
4221                                  &dev_priv->mm.inactive_list,
4222                                  mm_list) {
4223                 if (nr_to_scan &&
4224                     i915_gem_object_unbind(obj) == 0)
4225                         nr_to_scan--;
4226                 else
4227                         cnt++;
4228         }
4229
4230         if (nr_to_scan && i915_gpu_is_active(dev)) {
4231                 /*
4232                  * We are desperate for pages, so as a last resort, wait
4233                  * for the GPU to finish and discard whatever we can.
4234                  * This has a dramatic impact to reduce the number of
4235                  * OOM-killer events whilst running the GPU aggressively.
4236                  */
4237                 if (i915_gpu_idle(dev) == 0)
4238                         goto rescan;
4239         }
4240         mutex_unlock(&dev->struct_mutex);
4241         return cnt / 100 * sysctl_vfs_cache_pressure;
4242 }