drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  40
  41 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  42 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  43 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  44 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  45                                              int write);
  46 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  47                                                      uint64_t offset,
  48                                                      uint64_t size);
  49 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  50 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  51                                           bool interruptible);
  52 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  53                                            unsigned alignment);
  54 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  55 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  56                                 struct drm_i915_gem_pwrite *args,
  57                                 struct drm_file *file_priv);
  58 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  59
  60 static int
  61 i915_gem_object_get_pages(struct drm_gem_object *obj,
  62                           gfp_t gfpmask);
  63
  64 static void
  65 i915_gem_object_put_pages(struct drm_gem_object *obj);
  66
  67 static LIST_HEAD(shrink_list);
  68 static DEFINE_SPINLOCK(shrink_list_lock);
  69
  70 /* some bookkeeping */
  71 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  72                                   size_t size)
  73 {
  74         dev_priv->mm.object_count++;
  75         dev_priv->mm.object_memory += size;
  76 }
  77
  78 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  79                                      size_t size)
  80 {
  81         dev_priv->mm.object_count--;
  82         dev_priv->mm.object_memory -= size;
  83 }
  84
  85 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
  86                                   size_t size)
  87 {
  88         dev_priv->mm.gtt_count++;
  89         dev_priv->mm.gtt_memory += size;
  90 }
  91
  92 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
  93                                      size_t size)
  94 {
  95         dev_priv->mm.gtt_count--;
  96         dev_priv->mm.gtt_memory -= size;
  97 }
  98
  99 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
 100                                   size_t size)
 101 {
 102         dev_priv->mm.pin_count++;
 103         dev_priv->mm.pin_memory += size;
 104 }
 105
 106 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
 107                                      size_t size)
 108 {
 109         dev_priv->mm.pin_count--;
 110         dev_priv->mm.pin_memory -= size;
 111 }
 112
 113 int
 114 i915_gem_check_is_wedged(struct drm_device *dev)
 115 {
 116         struct drm_i915_private *dev_priv = dev->dev_private;
 117         struct completion *x = &dev_priv->error_completion;
 118         unsigned long flags;
 119         int ret;
 120
 121         if (!atomic_read(&dev_priv->mm.wedged))
 122                 return 0;
 123
 124         ret = wait_for_completion_interruptible(x);
 125         if (ret)
 126                 return ret;
 127
 128         /* Success, we reset the GPU! */
 129         if (!atomic_read(&dev_priv->mm.wedged))
 130                 return 0;
 131
 132         /* GPU is hung, bump the completion count to account for
 133          * the token we just consumed so that we never hit zero and
 134          * end up waiting upon a subsequent completion event that
 135          * will never happen.
 136          */
 137         spin_lock_irqsave(&x->wait.lock, flags);
 138         x->done++;
 139         spin_unlock_irqrestore(&x->wait.lock, flags);
 140         return -EIO;
 141 }
 142
 143 static int i915_mutex_lock_interruptible(struct drm_device *dev)
 144 {
 145         struct drm_i915_private *dev_priv = dev->dev_private;
 146         int ret;
 147
 148         ret = i915_gem_check_is_wedged(dev);
 149         if (ret)
 150                 return ret;
 151
 152         ret = mutex_lock_interruptible(&dev->struct_mutex);
 153         if (ret)
 154                 return ret;
 155
 156         if (atomic_read(&dev_priv->mm.wedged)) {
 157                 mutex_unlock(&dev->struct_mutex);
 158                 return -EAGAIN;
 159         }
 160
 161         WARN_ON(i915_verify_lists(dev));
 162         return 0;
 163 }
 164
 165 static inline bool
 166 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
 167 {
 168         return obj_priv->gtt_space &&
 169                 !obj_priv->active &&
 170                 obj_priv->pin_count == 0;
 171 }
 172
 173 int i915_gem_do_init(struct drm_device *dev,
 174                      unsigned long start,
 175                      unsigned long end)
 176 {
 177         drm_i915_private_t *dev_priv = dev->dev_private;
 178
 179         if (start >= end ||
 180             (start & (PAGE_SIZE - 1)) != 0 ||
 181             (end & (PAGE_SIZE - 1)) != 0) {
 182                 return -EINVAL;
 183         }
 184
 185         drm_mm_init(&dev_priv->mm.gtt_space, start,
 186                     end - start);
 187
 188         dev_priv->mm.gtt_total = end - start;
 189
 190         return 0;
 191 }
 192
 193 int
 194 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 195                     struct drm_file *file_priv)
 196 {
 197         struct drm_i915_gem_init *args = data;
 198         int ret;
 199
 200         mutex_lock(&dev->struct_mutex);
 201         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
 202         mutex_unlock(&dev->struct_mutex);
 203
 204         return ret;
 205 }
 206
 207 int
 208 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 209                             struct drm_file *file_priv)
 210 {
 211         struct drm_i915_private *dev_priv = dev->dev_private;
 212         struct drm_i915_gem_get_aperture *args = data;
 213
 214         if (!(dev->driver->driver_features & DRIVER_GEM))
 215                 return -ENODEV;
 216
 217         mutex_lock(&dev->struct_mutex);
 218         args->aper_size = dev_priv->mm.gtt_total;
 219         args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
 220         mutex_unlock(&dev->struct_mutex);
 221
 222         return 0;
 223 }
 224
 225
 226 /**
 227  * Creates a new mm object and returns a handle to it.
 228  */
 229 int
 230 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 231                       struct drm_file *file_priv)
 232 {
 233         struct drm_i915_gem_create *args = data;
 234         struct drm_gem_object *obj;
 235         int ret;
 236         u32 handle;
 237
 238         args->size = roundup(args->size, PAGE_SIZE);
 239
 240         /* Allocate the new object */
 241         obj = i915_gem_alloc_object(dev, args->size);
 242         if (obj == NULL)
 243                 return -ENOMEM;
 244
 245         ret = drm_gem_handle_create(file_priv, obj, &handle);
 246         if (ret) {
 247                 drm_gem_object_release(obj);
 248                 i915_gem_info_remove_obj(dev->dev_private, obj->size);
 249                 kfree(obj);
 250                 return ret;
 251         }
 252
 253         /* drop reference from allocate - handle holds it now */
 254         drm_gem_object_unreference(obj);
 255         trace_i915_gem_object_create(obj);
 256
 257         args->handle = handle;
 258         return 0;
 259 }
 260
 261 static inline int
 262 fast_shmem_read(struct page **pages,
 263                 loff_t page_base, int page_offset,
 264                 char __user *data,
 265                 int length)
 266 {
 267         char *vaddr;
 268         int ret;
 269
 270         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
 271         ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 272         kunmap_atomic(vaddr);
 273
 274         return ret;
 275 }
 276
 277 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 278 {
 279         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 280         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 281
 282         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 283                 obj_priv->tiling_mode != I915_TILING_NONE;
 284 }
 285
 286 static inline void
 287 slow_shmem_copy(struct page *dst_page,
 288                 int dst_offset,
 289                 struct page *src_page,
 290                 int src_offset,
 291                 int length)
 292 {
 293         char *dst_vaddr, *src_vaddr;
 294
 295         dst_vaddr = kmap(dst_page);
 296         src_vaddr = kmap(src_page);
 297
 298         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 299
 300         kunmap(src_page);
 301         kunmap(dst_page);
 302 }
 303
 304 static inline void
 305 slow_shmem_bit17_copy(struct page *gpu_page,
 306                       int gpu_offset,
 307                       struct page *cpu_page,
 308                       int cpu_offset,
 309                       int length,
 310                       int is_read)
 311 {
 312         char *gpu_vaddr, *cpu_vaddr;
 313
 314         /* Use the unswizzled path if this page isn't affected. */
 315         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 316                 if (is_read)
 317                         return slow_shmem_copy(cpu_page, cpu_offset,
 318                                                gpu_page, gpu_offset, length);
 319                 else
 320                         return slow_shmem_copy(gpu_page, gpu_offset,
 321                                                cpu_page, cpu_offset, length);
 322         }
 323
 324         gpu_vaddr = kmap(gpu_page);
 325         cpu_vaddr = kmap(cpu_page);
 326
 327         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 328          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 329          */
 330         while (length > 0) {
 331                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 332                 int this_length = min(cacheline_end - gpu_offset, length);
 333                 int swizzled_gpu_offset = gpu_offset ^ 64;
 334
 335                 if (is_read) {
 336                         memcpy(cpu_vaddr + cpu_offset,
 337                                gpu_vaddr + swizzled_gpu_offset,
 338                                this_length);
 339                 } else {
 340                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 341                                cpu_vaddr + cpu_offset,
 342                                this_length);
 343                 }
 344                 cpu_offset += this_length;
 345                 gpu_offset += this_length;
 346                 length -= this_length;
 347         }
 348
 349         kunmap(cpu_page);
 350         kunmap(gpu_page);
 351 }
 352
 353 /**
 354  * This is the fast shmem pread path, which attempts to copy_from_user directly
 355  * from the backing pages of the object to the user's address space.  On a
 356  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 357  */
 358 static int
 359 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 360                           struct drm_i915_gem_pread *args,
 361                           struct drm_file *file_priv)
 362 {
 363         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 364         ssize_t remain;
 365         loff_t offset, page_base;
 366         char __user *user_data;
 367         int page_offset, page_length;
 368
 369         user_data = (char __user *) (uintptr_t) args->data_ptr;
 370         remain = args->size;
 371
 372         obj_priv = to_intel_bo(obj);
 373         offset = args->offset;
 374
 375         while (remain > 0) {
 376                 /* Operation in this page
 377                  *
 378                  * page_base = page offset within aperture
 379                  * page_offset = offset within page
 380                  * page_length = bytes to copy for this page
 381                  */
 382                 page_base = (offset & ~(PAGE_SIZE-1));
 383                 page_offset = offset & (PAGE_SIZE-1);
 384                 page_length = remain;
 385                 if ((page_offset + remain) > PAGE_SIZE)
 386                         page_length = PAGE_SIZE - page_offset;
 387
 388                 if (fast_shmem_read(obj_priv->pages,
 389                                     page_base, page_offset,
 390                                     user_data, page_length))
 391                         return -EFAULT;
 392
 393                 remain -= page_length;
 394                 user_data += page_length;
 395                 offset += page_length;
 396         }
 397
 398         return 0;
 399 }
 400
 401 static int
 402 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 403 {
 404         int ret;
 405
 406         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 407
 408         /* If we've insufficient memory to map in the pages, attempt
 409          * to make some space by throwing out some old buffers.
 410          */
 411         if (ret == -ENOMEM) {
 412                 struct drm_device *dev = obj->dev;
 413
 414                 ret = i915_gem_evict_something(dev, obj->size,
 415                                                i915_gem_get_gtt_alignment(obj));
 416                 if (ret)
 417                         return ret;
 418
 419                 ret = i915_gem_object_get_pages(obj, 0);
 420         }
 421
 422         return ret;
 423 }
 424
 425 /**
 426  * This is the fallback shmem pread path, which allocates temporary storage
 427  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 428  * can copy out of the object's backing pages while holding the struct mutex
 429  * and not take page faults.
 430  */
 431 static int
 432 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 433                           struct drm_i915_gem_pread *args,
 434                           struct drm_file *file_priv)
 435 {
 436         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 437         struct mm_struct *mm = current->mm;
 438         struct page **user_pages;
 439         ssize_t remain;
 440         loff_t offset, pinned_pages, i;
 441         loff_t first_data_page, last_data_page, num_pages;
 442         int shmem_page_index, shmem_page_offset;
 443         int data_page_index,  data_page_offset;
 444         int page_length;
 445         int ret;
 446         uint64_t data_ptr = args->data_ptr;
 447         int do_bit17_swizzling;
 448
 449         remain = args->size;
 450
 451         /* Pin the user pages containing the data.  We can't fault while
 452          * holding the struct mutex, yet we want to hold it while
 453          * dereferencing the user data.
 454          */
 455         first_data_page = data_ptr / PAGE_SIZE;
 456         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 457         num_pages = last_data_page - first_data_page + 1;
 458
 459         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 460         if (user_pages == NULL)
 461                 return -ENOMEM;
 462
 463         mutex_unlock(&dev->struct_mutex);
 464         down_read(&mm->mmap_sem);
 465         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 466                                       num_pages, 1, 0, user_pages, NULL);
 467         up_read(&mm->mmap_sem);
 468         mutex_lock(&dev->struct_mutex);
 469         if (pinned_pages < num_pages) {
 470                 ret = -EFAULT;
 471                 goto out;
 472         }
 473
 474         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 475                                                         args->offset,
 476                                                         args->size);
 477         if (ret)
 478                 goto out;
 479
 480         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 481
 482         obj_priv = to_intel_bo(obj);
 483         offset = args->offset;
 484
 485         while (remain > 0) {
 486                 /* Operation in this page
 487                  *
 488                  * shmem_page_index = page number within shmem file
 489                  * shmem_page_offset = offset within page in shmem file
 490                  * data_page_index = page number in get_user_pages return
 491                  * data_page_offset = offset with data_page_index page.
 492                  * page_length = bytes to copy for this page
 493                  */
 494                 shmem_page_index = offset / PAGE_SIZE;
 495                 shmem_page_offset = offset & ~PAGE_MASK;
 496                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 497                 data_page_offset = data_ptr & ~PAGE_MASK;
 498
 499                 page_length = remain;
 500                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 501                         page_length = PAGE_SIZE - shmem_page_offset;
 502                 if ((data_page_offset + page_length) > PAGE_SIZE)
 503                         page_length = PAGE_SIZE - data_page_offset;
 504
 505                 if (do_bit17_swizzling) {
 506                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 507                                               shmem_page_offset,
 508                                               user_pages[data_page_index],
 509                                               data_page_offset,
 510                                               page_length,
 511                                               1);
 512                 } else {
 513                         slow_shmem_copy(user_pages[data_page_index],
 514                                         data_page_offset,
 515                                         obj_priv->pages[shmem_page_index],
 516                                         shmem_page_offset,
 517                                         page_length);
 518                 }
 519
 520                 remain -= page_length;
 521                 data_ptr += page_length;
 522                 offset += page_length;
 523         }
 524
 525 out:
 526         for (i = 0; i < pinned_pages; i++) {
 527                 SetPageDirty(user_pages[i]);
 528                 page_cache_release(user_pages[i]);
 529         }
 530         drm_free_large(user_pages);
 531
 532         return ret;
 533 }
 534
 535 /**
 536  * Reads data from the object referenced by handle.
 537  *
 538  * On error, the contents of *data are undefined.
 539  */
 540 int
 541 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 542                      struct drm_file *file_priv)
 543 {
 544         struct drm_i915_gem_pread *args = data;
 545         struct drm_gem_object *obj;
 546         struct drm_i915_gem_object *obj_priv;
 547         int ret = 0;
 548
 549         if (args->size == 0)
 550                 return 0;
 551
 552         if (!access_ok(VERIFY_WRITE,
 553                        (char __user *)(uintptr_t)args->data_ptr,
 554                        args->size))
 555                 return -EFAULT;
 556
 557         ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
 558                                        args->size);
 559         if (ret)
 560                 return -EFAULT;
 561
 562         ret = i915_mutex_lock_interruptible(dev);
 563         if (ret)
 564                 return ret;
 565
 566         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 567         if (obj == NULL) {
 568                 ret = -ENOENT;
 569                 goto unlock;
 570         }
 571         obj_priv = to_intel_bo(obj);
 572
 573         /* Bounds check source.  */
 574         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 575                 ret = -EINVAL;
 576                 goto out;
 577         }
 578
 579         ret = i915_gem_object_get_pages_or_evict(obj);
 580         if (ret)
 581                 goto out;
 582
 583         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 584                                                         args->offset,
 585                                                         args->size);
 586         if (ret)
 587                 goto out_put;
 588
 589         ret = -EFAULT;
 590         if (!i915_gem_object_needs_bit17_swizzle(obj))
 591                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 592         if (ret == -EFAULT)
 593                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 594
 595 out_put:
 596         i915_gem_object_put_pages(obj);
 597 out:
 598         drm_gem_object_unreference(obj);
 599 unlock:
 600         mutex_unlock(&dev->struct_mutex);
 601         return ret;
 602 }
 603
 604 /* This is the fast write path which cannot handle
 605  * page faults in the source data
 606  */
 607
 608 static inline int
 609 fast_user_write(struct io_mapping *mapping,
 610                 loff_t page_base, int page_offset,
 611                 char __user *user_data,
 612                 int length)
 613 {
 614         char *vaddr_atomic;
 615         unsigned long unwritten;
 616
 617         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 618         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 619                                                       user_data, length);
 620         io_mapping_unmap_atomic(vaddr_atomic);
 621         return unwritten;
 622 }
 623
 624 /* Here's the write path which can sleep for
 625  * page faults
 626  */
 627
 628 static inline void
 629 slow_kernel_write(struct io_mapping *mapping,
 630                   loff_t gtt_base, int gtt_offset,
 631                   struct page *user_page, int user_offset,
 632                   int length)
 633 {
 634         char __iomem *dst_vaddr;
 635         char *src_vaddr;
 636
 637         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 638         src_vaddr = kmap(user_page);
 639
 640         memcpy_toio(dst_vaddr + gtt_offset,
 641                     src_vaddr + user_offset,
 642                     length);
 643
 644         kunmap(user_page);
 645         io_mapping_unmap(dst_vaddr);
 646 }
 647
 648 static inline int
 649 fast_shmem_write(struct page **pages,
 650                  loff_t page_base, int page_offset,
 651                  char __user *data,
 652                  int length)
 653 {
 654         char *vaddr;
 655         int ret;
 656
 657         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]);
 658         ret = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 659         kunmap_atomic(vaddr);
 660
 661         return ret;
 662 }
 663
 664 /**
 665  * This is the fast pwrite path, where we copy the data directly from the
 666  * user into the GTT, uncached.
 667  */
 668 static int
 669 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 670                          struct drm_i915_gem_pwrite *args,
 671                          struct drm_file *file_priv)
 672 {
 673         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 674         drm_i915_private_t *dev_priv = dev->dev_private;
 675         ssize_t remain;
 676         loff_t offset, page_base;
 677         char __user *user_data;
 678         int page_offset, page_length;
 679
 680         user_data = (char __user *) (uintptr_t) args->data_ptr;
 681         remain = args->size;
 682
 683         obj_priv = to_intel_bo(obj);
 684         offset = obj_priv->gtt_offset + args->offset;
 685
 686         while (remain > 0) {
 687                 /* Operation in this page
 688                  *
 689                  * page_base = page offset within aperture
 690                  * page_offset = offset within page
 691                  * page_length = bytes to copy for this page
 692                  */
 693                 page_base = (offset & ~(PAGE_SIZE-1));
 694                 page_offset = offset & (PAGE_SIZE-1);
 695                 page_length = remain;
 696                 if ((page_offset + remain) > PAGE_SIZE)
 697                         page_length = PAGE_SIZE - page_offset;
 698
 699                 /* If we get a fault while copying data, then (presumably) our
 700                  * source page isn't available.  Return the error and we'll
 701                  * retry in the slow path.
 702                  */
 703                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 704                                     page_offset, user_data, page_length))
 705
 706                         return -EFAULT;
 707
 708                 remain -= page_length;
 709                 user_data += page_length;
 710                 offset += page_length;
 711         }
 712
 713         return 0;
 714 }
 715
 716 /**
 717  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 718  * the memory and maps it using kmap_atomic for copying.
 719  *
 720  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 721  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 722  */
 723 static int
 724 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 725                          struct drm_i915_gem_pwrite *args,
 726                          struct drm_file *file_priv)
 727 {
 728         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 729         drm_i915_private_t *dev_priv = dev->dev_private;
 730         ssize_t remain;
 731         loff_t gtt_page_base, offset;
 732         loff_t first_data_page, last_data_page, num_pages;
 733         loff_t pinned_pages, i;
 734         struct page **user_pages;
 735         struct mm_struct *mm = current->mm;
 736         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 737         int ret;
 738         uint64_t data_ptr = args->data_ptr;
 739
 740         remain = args->size;
 741
 742         /* Pin the user pages containing the data.  We can't fault while
 743          * holding the struct mutex, and all of the pwrite implementations
 744          * want to hold it while dereferencing the user data.
 745          */
 746         first_data_page = data_ptr / PAGE_SIZE;
 747         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 748         num_pages = last_data_page - first_data_page + 1;
 749
 750         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 751         if (user_pages == NULL)
 752                 return -ENOMEM;
 753
 754         mutex_unlock(&dev->struct_mutex);
 755         down_read(&mm->mmap_sem);
 756         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 757                                       num_pages, 0, 0, user_pages, NULL);
 758         up_read(&mm->mmap_sem);
 759         mutex_lock(&dev->struct_mutex);
 760         if (pinned_pages < num_pages) {
 761                 ret = -EFAULT;
 762                 goto out_unpin_pages;
 763         }
 764
 765         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 766         if (ret)
 767                 goto out_unpin_pages;
 768
 769         obj_priv = to_intel_bo(obj);
 770         offset = obj_priv->gtt_offset + args->offset;
 771
 772         while (remain > 0) {
 773                 /* Operation in this page
 774                  *
 775                  * gtt_page_base = page offset within aperture
 776                  * gtt_page_offset = offset within page in aperture
 777                  * data_page_index = page number in get_user_pages return
 778                  * data_page_offset = offset with data_page_index page.
 779                  * page_length = bytes to copy for this page
 780                  */
 781                 gtt_page_base = offset & PAGE_MASK;
 782                 gtt_page_offset = offset & ~PAGE_MASK;
 783                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 784                 data_page_offset = data_ptr & ~PAGE_MASK;
 785
 786                 page_length = remain;
 787                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 788                         page_length = PAGE_SIZE - gtt_page_offset;
 789                 if ((data_page_offset + page_length) > PAGE_SIZE)
 790                         page_length = PAGE_SIZE - data_page_offset;
 791
 792                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 793                                   gtt_page_base, gtt_page_offset,
 794                                   user_pages[data_page_index],
 795                                   data_page_offset,
 796                                   page_length);
 797
 798                 remain -= page_length;
 799                 offset += page_length;
 800                 data_ptr += page_length;
 801         }
 802
 803 out_unpin_pages:
 804         for (i = 0; i < pinned_pages; i++)
 805                 page_cache_release(user_pages[i]);
 806         drm_free_large(user_pages);
 807
 808         return ret;
 809 }
 810
 811 /**
 812  * This is the fast shmem pwrite path, which attempts to directly
 813  * copy_from_user into the kmapped pages backing the object.
 814  */
 815 static int
 816 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 817                            struct drm_i915_gem_pwrite *args,
 818                            struct drm_file *file_priv)
 819 {
 820         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 821         ssize_t remain;
 822         loff_t offset, page_base;
 823         char __user *user_data;
 824         int page_offset, page_length;
 825
 826         user_data = (char __user *) (uintptr_t) args->data_ptr;
 827         remain = args->size;
 828
 829         obj_priv = to_intel_bo(obj);
 830         offset = args->offset;
 831         obj_priv->dirty = 1;
 832
 833         while (remain > 0) {
 834                 /* Operation in this page
 835                  *
 836                  * page_base = page offset within aperture
 837                  * page_offset = offset within page
 838                  * page_length = bytes to copy for this page
 839                  */
 840                 page_base = (offset & ~(PAGE_SIZE-1));
 841                 page_offset = offset & (PAGE_SIZE-1);
 842                 page_length = remain;
 843                 if ((page_offset + remain) > PAGE_SIZE)
 844                         page_length = PAGE_SIZE - page_offset;
 845
 846                 if (fast_shmem_write(obj_priv->pages,
 847                                        page_base, page_offset,
 848                                        user_data, page_length))
 849                         return -EFAULT;
 850
 851                 remain -= page_length;
 852                 user_data += page_length;
 853                 offset += page_length;
 854         }
 855
 856         return 0;
 857 }
 858
 859 /**
 860  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 861  * the memory and maps it using kmap_atomic for copying.
 862  *
 863  * This avoids taking mmap_sem for faulting on the user's address while the
 864  * struct_mutex is held.
 865  */
 866 static int
 867 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 868                            struct drm_i915_gem_pwrite *args,
 869                            struct drm_file *file_priv)
 870 {
 871         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 872         struct mm_struct *mm = current->mm;
 873         struct page **user_pages;
 874         ssize_t remain;
 875         loff_t offset, pinned_pages, i;
 876         loff_t first_data_page, last_data_page, num_pages;
 877         int shmem_page_index, shmem_page_offset;
 878         int data_page_index,  data_page_offset;
 879         int page_length;
 880         int ret;
 881         uint64_t data_ptr = args->data_ptr;
 882         int do_bit17_swizzling;
 883
 884         remain = args->size;
 885
 886         /* Pin the user pages containing the data.  We can't fault while
 887          * holding the struct mutex, and all of the pwrite implementations
 888          * want to hold it while dereferencing the user data.
 889          */
 890         first_data_page = data_ptr / PAGE_SIZE;
 891         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 892         num_pages = last_data_page - first_data_page + 1;
 893
 894         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 895         if (user_pages == NULL)
 896                 return -ENOMEM;
 897
 898         mutex_unlock(&dev->struct_mutex);
 899         down_read(&mm->mmap_sem);
 900         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 901                                       num_pages, 0, 0, user_pages, NULL);
 902         up_read(&mm->mmap_sem);
 903         mutex_lock(&dev->struct_mutex);
 904         if (pinned_pages < num_pages) {
 905                 ret = -EFAULT;
 906                 goto out;
 907         }
 908
 909         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 910         if (ret)
 911                 goto out;
 912
 913         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 914
 915         obj_priv = to_intel_bo(obj);
 916         offset = args->offset;
 917         obj_priv->dirty = 1;
 918
 919         while (remain > 0) {
 920                 /* Operation in this page
 921                  *
 922                  * shmem_page_index = page number within shmem file
 923                  * shmem_page_offset = offset within page in shmem file
 924                  * data_page_index = page number in get_user_pages return
 925                  * data_page_offset = offset with data_page_index page.
 926                  * page_length = bytes to copy for this page
 927                  */
 928                 shmem_page_index = offset / PAGE_SIZE;
 929                 shmem_page_offset = offset & ~PAGE_MASK;
 930                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 931                 data_page_offset = data_ptr & ~PAGE_MASK;
 932
 933                 page_length = remain;
 934                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 935                         page_length = PAGE_SIZE - shmem_page_offset;
 936                 if ((data_page_offset + page_length) > PAGE_SIZE)
 937                         page_length = PAGE_SIZE - data_page_offset;
 938
 939                 if (do_bit17_swizzling) {
 940                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 941                                               shmem_page_offset,
 942                                               user_pages[data_page_index],
 943                                               data_page_offset,
 944                                               page_length,
 945                                               0);
 946                 } else {
 947                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
 948                                         shmem_page_offset,
 949                                         user_pages[data_page_index],
 950                                         data_page_offset,
 951                                         page_length);
 952                 }
 953
 954                 remain -= page_length;
 955                 data_ptr += page_length;
 956                 offset += page_length;
 957         }
 958
 959 out:
 960         for (i = 0; i < pinned_pages; i++)
 961                 page_cache_release(user_pages[i]);
 962         drm_free_large(user_pages);
 963
 964         return ret;
 965 }
 966
 967 /**
 968  * Writes data to the object referenced by handle.
 969  *
 970  * On error, the contents of the buffer that were to be modified are undefined.
 971  */
 972 int
 973 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 974                       struct drm_file *file)
 975 {
 976         struct drm_i915_gem_pwrite *args = data;
 977         struct drm_gem_object *obj;
 978         struct drm_i915_gem_object *obj_priv;
 979         int ret;
 980
 981         if (args->size == 0)
 982                 return 0;
 983
 984         if (!access_ok(VERIFY_READ,
 985                        (char __user *)(uintptr_t)args->data_ptr,
 986                        args->size))
 987                 return -EFAULT;
 988
 989         ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
 990                                       args->size);
 991         if (ret)
 992                 return -EFAULT;
 993
 994         ret = i915_mutex_lock_interruptible(dev);
 995         if (ret)
 996                 return ret;
 997
 998         obj = drm_gem_object_lookup(dev, file, args->handle);
 999         if (obj == NULL) {
1000                 ret = -ENOENT;
1001                 goto unlock;
1002         }
1003         obj_priv = to_intel_bo(obj);
1004
1005         /* Bounds check destination. */
1006         if (args->offset > obj->size || args->size > obj->size - args->offset) {
1007                 ret = -EINVAL;
1008                 goto out;
1009         }
1010
1011         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1012          * it would end up going through the fenced access, and we'll get
1013          * different detiling behavior between reading and writing.
1014          * pread/pwrite currently are reading and writing from the CPU
1015          * perspective, requiring manual detiling by the client.
1016          */
1017         if (obj_priv->phys_obj)
1018                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1019         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1020                  obj_priv->gtt_space &&
1021                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
1022                 ret = i915_gem_object_pin(obj, 0);
1023                 if (ret)
1024                         goto out;
1025
1026                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1027                 if (ret)
1028                         goto out_unpin;
1029
1030                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1031                 if (ret == -EFAULT)
1032                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1033
1034 out_unpin:
1035                 i915_gem_object_unpin(obj);
1036         } else {
1037                 ret = i915_gem_object_get_pages_or_evict(obj);
1038                 if (ret)
1039                         goto out;
1040
1041                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1042                 if (ret)
1043                         goto out_put;
1044
1045                 ret = -EFAULT;
1046                 if (!i915_gem_object_needs_bit17_swizzle(obj))
1047                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1048                 if (ret == -EFAULT)
1049                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1050
1051 out_put:
1052                 i915_gem_object_put_pages(obj);
1053         }
1054
1055 out:
1056         drm_gem_object_unreference(obj);
1057 unlock:
1058         mutex_unlock(&dev->struct_mutex);
1059         return ret;
1060 }
1061
1062 /**
1063  * Called when user space prepares to use an object with the CPU, either
1064  * through the mmap ioctl's mapping or a GTT mapping.
1065  */
1066 int
1067 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1068                           struct drm_file *file_priv)
1069 {
1070         struct drm_i915_private *dev_priv = dev->dev_private;
1071         struct drm_i915_gem_set_domain *args = data;
1072         struct drm_gem_object *obj;
1073         struct drm_i915_gem_object *obj_priv;
1074         uint32_t read_domains = args->read_domains;
1075         uint32_t write_domain = args->write_domain;
1076         int ret;
1077
1078         if (!(dev->driver->driver_features & DRIVER_GEM))
1079                 return -ENODEV;
1080
1081         /* Only handle setting domains to types used by the CPU. */
1082         if (write_domain & I915_GEM_GPU_DOMAINS)
1083                 return -EINVAL;
1084
1085         if (read_domains & I915_GEM_GPU_DOMAINS)
1086                 return -EINVAL;
1087
1088         /* Having something in the write domain implies it's in the read
1089          * domain, and only that read domain.  Enforce that in the request.
1090          */
1091         if (write_domain != 0 && read_domains != write_domain)
1092                 return -EINVAL;
1093
1094         ret = i915_mutex_lock_interruptible(dev);
1095         if (ret)
1096                 return ret;
1097
1098         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1099         if (obj == NULL) {
1100                 ret = -ENOENT;
1101                 goto unlock;
1102         }
1103         obj_priv = to_intel_bo(obj);
1104
1105         intel_mark_busy(dev, obj);
1106
1107         if (read_domains & I915_GEM_DOMAIN_GTT) {
1108                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1109
1110                 /* Update the LRU on the fence for the CPU access that's
1111                  * about to occur.
1112                  */
1113                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1114                         struct drm_i915_fence_reg *reg =
1115                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1116                         list_move_tail(&reg->lru_list,
1117                                        &dev_priv->mm.fence_list);
1118                 }
1119
1120                 /* Silently promote "you're not bound, there was nothing to do"
1121                  * to success, since the client was just asking us to
1122                  * make sure everything was done.
1123                  */
1124                 if (ret == -EINVAL)
1125                         ret = 0;
1126         } else {
1127                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1128         }
1129
1130         /* Maintain LRU order of "inactive" objects */
1131         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1132                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1133
1134         drm_gem_object_unreference(obj);
1135 unlock:
1136         mutex_unlock(&dev->struct_mutex);
1137         return ret;
1138 }
1139
1140 /**
1141  * Called when user space has done writes to this buffer
1142  */
1143 int
1144 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1145                       struct drm_file *file_priv)
1146 {
1147         struct drm_i915_gem_sw_finish *args = data;
1148         struct drm_gem_object *obj;
1149         int ret = 0;
1150
1151         if (!(dev->driver->driver_features & DRIVER_GEM))
1152                 return -ENODEV;
1153
1154         ret = i915_mutex_lock_interruptible(dev);
1155         if (ret)
1156                 return ret;
1157
1158         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1159         if (obj == NULL) {
1160                 ret = -ENOENT;
1161                 goto unlock;
1162         }
1163
1164         /* Pinned buffers may be scanout, so flush the cache */
1165         if (to_intel_bo(obj)->pin_count)
1166                 i915_gem_object_flush_cpu_write_domain(obj);
1167
1168         drm_gem_object_unreference(obj);
1169 unlock:
1170         mutex_unlock(&dev->struct_mutex);
1171         return ret;
1172 }
1173
1174 /**
1175  * Maps the contents of an object, returning the address it is mapped
1176  * into.
1177  *
1178  * While the mapping holds a reference on the contents of the object, it doesn't
1179  * imply a ref on the object itself.
1180  */
1181 int
1182 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1183                    struct drm_file *file_priv)
1184 {
1185         struct drm_i915_gem_mmap *args = data;
1186         struct drm_gem_object *obj;
1187         loff_t offset;
1188         unsigned long addr;
1189
1190         if (!(dev->driver->driver_features & DRIVER_GEM))
1191                 return -ENODEV;
1192
1193         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1194         if (obj == NULL)
1195                 return -ENOENT;
1196
1197         offset = args->offset;
1198
1199         down_write(&current->mm->mmap_sem);
1200         addr = do_mmap(obj->filp, 0, args->size,
1201                        PROT_READ | PROT_WRITE, MAP_SHARED,
1202                        args->offset);
1203         up_write(&current->mm->mmap_sem);
1204         drm_gem_object_unreference_unlocked(obj);
1205         if (IS_ERR((void *)addr))
1206                 return addr;
1207
1208         args->addr_ptr = (uint64_t) addr;
1209
1210         return 0;
1211 }
1212
1213 /**
1214  * i915_gem_fault - fault a page into the GTT
1215  * vma: VMA in question
1216  * vmf: fault info
1217  *
1218  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1219  * from userspace.  The fault handler takes care of binding the object to
1220  * the GTT (if needed), allocating and programming a fence register (again,
1221  * only if needed based on whether the old reg is still valid or the object
1222  * is tiled) and inserting a new PTE into the faulting process.
1223  *
1224  * Note that the faulting process may involve evicting existing objects
1225  * from the GTT and/or fence registers to make room.  So performance may
1226  * suffer if the GTT working set is large or there are few fence registers
1227  * left.
1228  */
1229 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1230 {
1231         struct drm_gem_object *obj = vma->vm_private_data;
1232         struct drm_device *dev = obj->dev;
1233         drm_i915_private_t *dev_priv = dev->dev_private;
1234         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1235         pgoff_t page_offset;
1236         unsigned long pfn;
1237         int ret = 0;
1238         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1239
1240         /* We don't use vmf->pgoff since that has the fake offset */
1241         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1242                 PAGE_SHIFT;
1243
1244         /* Now bind it into the GTT if needed */
1245         mutex_lock(&dev->struct_mutex);
1246         if (!obj_priv->gtt_space) {
1247                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1248                 if (ret)
1249                         goto unlock;
1250
1251                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1252                 if (ret)
1253                         goto unlock;
1254         }
1255
1256         /* Need a new fence register? */
1257         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1258                 ret = i915_gem_object_get_fence_reg(obj, true);
1259                 if (ret)
1260                         goto unlock;
1261         }
1262
1263         if (i915_gem_object_is_inactive(obj_priv))
1264                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1265
1266         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1267                 page_offset;
1268
1269         /* Finally, remap it using the new GTT offset */
1270         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1271 unlock:
1272         mutex_unlock(&dev->struct_mutex);
1273
1274         switch (ret) {
1275         case 0:
1276         case -ERESTARTSYS:
1277                 return VM_FAULT_NOPAGE;
1278         case -ENOMEM:
1279         case -EAGAIN:
1280                 return VM_FAULT_OOM;
1281         default:
1282                 return VM_FAULT_SIGBUS;
1283         }
1284 }
1285
1286 /**
1287  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1288  * @obj: obj in question
1289  *
1290  * GEM memory mapping works by handing back to userspace a fake mmap offset
1291  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1292  * up the object based on the offset and sets up the various memory mapping
1293  * structures.
1294  *
1295  * This routine allocates and attaches a fake offset for @obj.
1296  */
1297 static int
1298 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1299 {
1300         struct drm_device *dev = obj->dev;
1301         struct drm_gem_mm *mm = dev->mm_private;
1302         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1303         struct drm_map_list *list;
1304         struct drm_local_map *map;
1305         int ret = 0;
1306
1307         /* Set the object up for mmap'ing */
1308         list = &obj->map_list;
1309         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1310         if (!list->map)
1311                 return -ENOMEM;
1312
1313         map = list->map;
1314         map->type = _DRM_GEM;
1315         map->size = obj->size;
1316         map->handle = obj;
1317
1318         /* Get a DRM GEM mmap offset allocated... */
1319         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1320                                                     obj->size / PAGE_SIZE, 0, 0);
1321         if (!list->file_offset_node) {
1322                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1323                 ret = -ENOSPC;
1324                 goto out_free_list;
1325         }
1326
1327         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1328                                                   obj->size / PAGE_SIZE, 0);
1329         if (!list->file_offset_node) {
1330                 ret = -ENOMEM;
1331                 goto out_free_list;
1332         }
1333
1334         list->hash.key = list->file_offset_node->start;
1335         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1336         if (ret) {
1337                 DRM_ERROR("failed to add to map hash\n");
1338                 goto out_free_mm;
1339         }
1340
1341         /* By now we should be all set, any drm_mmap request on the offset
1342          * below will get to our mmap & fault handler */
1343         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1344
1345         return 0;
1346
1347 out_free_mm:
1348         drm_mm_put_block(list->file_offset_node);
1349 out_free_list:
1350         kfree(list->map);
1351
1352         return ret;
1353 }
1354
1355 /**
1356  * i915_gem_release_mmap - remove physical page mappings
1357  * @obj: obj in question
1358  *
1359  * Preserve the reservation of the mmapping with the DRM core code, but
1360  * relinquish ownership of the pages back to the system.
1361  *
1362  * It is vital that we remove the page mapping if we have mapped a tiled
1363  * object through the GTT and then lose the fence register due to
1364  * resource pressure. Similarly if the object has been moved out of the
1365  * aperture, than pages mapped into userspace must be revoked. Removing the
1366  * mapping will then trigger a page fault on the next user access, allowing
1367  * fixup by i915_gem_fault().
1368  */
1369 void
1370 i915_gem_release_mmap(struct drm_gem_object *obj)
1371 {
1372         struct drm_device *dev = obj->dev;
1373         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1374
1375         if (dev->dev_mapping)
1376                 unmap_mapping_range(dev->dev_mapping,
1377                                     obj_priv->mmap_offset, obj->size, 1);
1378 }
1379
1380 static void
1381 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1382 {
1383         struct drm_device *dev = obj->dev;
1384         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1385         struct drm_gem_mm *mm = dev->mm_private;
1386         struct drm_map_list *list;
1387
1388         list = &obj->map_list;
1389         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1390
1391         if (list->file_offset_node) {
1392                 drm_mm_put_block(list->file_offset_node);
1393                 list->file_offset_node = NULL;
1394         }
1395
1396         if (list->map) {
1397                 kfree(list->map);
1398                 list->map = NULL;
1399         }
1400
1401         obj_priv->mmap_offset = 0;
1402 }
1403
1404 /**
1405  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1406  * @obj: object to check
1407  *
1408  * Return the required GTT alignment for an object, taking into account
1409  * potential fence register mapping if needed.
1410  */
1411 static uint32_t
1412 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1413 {
1414         struct drm_device *dev = obj->dev;
1415         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1416         int start, i;
1417
1418         /*
1419          * Minimum alignment is 4k (GTT page size), but might be greater
1420          * if a fence register is needed for the object.
1421          */
1422         if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
1423                 return 4096;
1424
1425         /*
1426          * Previous chips need to be aligned to the size of the smallest
1427          * fence register that can contain the object.
1428          */
1429         if (INTEL_INFO(dev)->gen == 3)
1430                 start = 1024*1024;
1431         else
1432                 start = 512*1024;
1433
1434         for (i = start; i < obj->size; i <<= 1)
1435                 ;
1436
1437         return i;
1438 }
1439
1440 /**
1441  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1442  * @dev: DRM device
1443  * @data: GTT mapping ioctl data
1444  * @file_priv: GEM object info
1445  *
1446  * Simply returns the fake offset to userspace so it can mmap it.
1447  * The mmap call will end up in drm_gem_mmap(), which will set things
1448  * up so we can get faults in the handler above.
1449  *
1450  * The fault handler will take care of binding the object into the GTT
1451  * (since it may have been evicted to make room for something), allocating
1452  * a fence register, and mapping the appropriate aperture address into
1453  * userspace.
1454  */
1455 int
1456 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1457                         struct drm_file *file_priv)
1458 {
1459         struct drm_i915_gem_mmap_gtt *args = data;
1460         struct drm_gem_object *obj;
1461         struct drm_i915_gem_object *obj_priv;
1462         int ret;
1463
1464         if (!(dev->driver->driver_features & DRIVER_GEM))
1465                 return -ENODEV;
1466
1467         ret = i915_mutex_lock_interruptible(dev);
1468         if (ret)
1469                 return ret;
1470
1471         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1472         if (obj == NULL) {
1473                 ret = -ENOENT;
1474                 goto unlock;
1475         }
1476         obj_priv = to_intel_bo(obj);
1477
1478         if (obj_priv->madv != I915_MADV_WILLNEED) {
1479                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1480                 ret = -EINVAL;
1481                 goto out;
1482         }
1483
1484         if (!obj_priv->mmap_offset) {
1485                 ret = i915_gem_create_mmap_offset(obj);
1486                 if (ret)
1487                         goto out;
1488         }
1489
1490         args->offset = obj_priv->mmap_offset;
1491
1492         /*
1493          * Pull it into the GTT so that we have a page list (makes the
1494          * initial fault faster and any subsequent flushing possible).
1495          */
1496         if (!obj_priv->agp_mem) {
1497                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1498                 if (ret)
1499                         goto out;
1500         }
1501
1502 out:
1503         drm_gem_object_unreference(obj);
1504 unlock:
1505         mutex_unlock(&dev->struct_mutex);
1506         return ret;
1507 }
1508
1509 static void
1510 i915_gem_object_put_pages(struct drm_gem_object *obj)
1511 {
1512         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1513         int page_count = obj->size / PAGE_SIZE;
1514         int i;
1515
1516         BUG_ON(obj_priv->pages_refcount == 0);
1517         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1518
1519         if (--obj_priv->pages_refcount != 0)
1520                 return;
1521
1522         if (obj_priv->tiling_mode != I915_TILING_NONE)
1523                 i915_gem_object_save_bit_17_swizzle(obj);
1524
1525         if (obj_priv->madv == I915_MADV_DONTNEED)
1526                 obj_priv->dirty = 0;
1527
1528         for (i = 0; i < page_count; i++) {
1529                 if (obj_priv->dirty)
1530                         set_page_dirty(obj_priv->pages[i]);
1531
1532                 if (obj_priv->madv == I915_MADV_WILLNEED)
1533                         mark_page_accessed(obj_priv->pages[i]);
1534
1535                 page_cache_release(obj_priv->pages[i]);
1536         }
1537         obj_priv->dirty = 0;
1538
1539         drm_free_large(obj_priv->pages);
1540         obj_priv->pages = NULL;
1541 }
1542
1543 static uint32_t
1544 i915_gem_next_request_seqno(struct drm_device *dev,
1545                             struct intel_ring_buffer *ring)
1546 {
1547         drm_i915_private_t *dev_priv = dev->dev_private;
1548
1549         ring->outstanding_lazy_request = true;
1550         return dev_priv->next_seqno;
1551 }
1552
1553 static void
1554 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1555                                struct intel_ring_buffer *ring)
1556 {
1557         struct drm_device *dev = obj->dev;
1558         struct drm_i915_private *dev_priv = dev->dev_private;
1559         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1560         uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1561
1562         BUG_ON(ring == NULL);
1563         obj_priv->ring = ring;
1564
1565         /* Add a reference if we're newly entering the active list. */
1566         if (!obj_priv->active) {
1567                 drm_gem_object_reference(obj);
1568                 obj_priv->active = 1;
1569         }
1570
1571         /* Move from whatever list we were on to the tail of execution. */
1572         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1573         list_move_tail(&obj_priv->ring_list, &ring->active_list);
1574         obj_priv->last_rendering_seqno = seqno;
1575 }
1576
1577 static void
1578 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1579 {
1580         struct drm_device *dev = obj->dev;
1581         drm_i915_private_t *dev_priv = dev->dev_private;
1582         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1583
1584         BUG_ON(!obj_priv->active);
1585         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1586         list_del_init(&obj_priv->ring_list);
1587         obj_priv->last_rendering_seqno = 0;
1588 }
1589
1590 /* Immediately discard the backing storage */
1591 static void
1592 i915_gem_object_truncate(struct drm_gem_object *obj)
1593 {
1594         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1595         struct inode *inode;
1596
1597         /* Our goal here is to return as much of the memory as
1598          * is possible back to the system as we are called from OOM.
1599          * To do this we must instruct the shmfs to drop all of its
1600          * backing pages, *now*. Here we mirror the actions taken
1601          * when by shmem_delete_inode() to release the backing store.
1602          */
1603         inode = obj->filp->f_path.dentry->d_inode;
1604         truncate_inode_pages(inode->i_mapping, 0);
1605         if (inode->i_op->truncate_range)
1606                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1607
1608         obj_priv->madv = __I915_MADV_PURGED;
1609 }
1610
1611 static inline int
1612 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1613 {
1614         return obj_priv->madv == I915_MADV_DONTNEED;
1615 }
1616
1617 static void
1618 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1619 {
1620         struct drm_device *dev = obj->dev;
1621         drm_i915_private_t *dev_priv = dev->dev_private;
1622         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1623
1624         if (obj_priv->pin_count != 0)
1625                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1626         else
1627                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1628         list_del_init(&obj_priv->ring_list);
1629
1630         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1631
1632         obj_priv->last_rendering_seqno = 0;
1633         obj_priv->ring = NULL;
1634         if (obj_priv->active) {
1635                 obj_priv->active = 0;
1636                 drm_gem_object_unreference(obj);
1637         }
1638         WARN_ON(i915_verify_lists(dev));
1639 }
1640
1641 static void
1642 i915_gem_process_flushing_list(struct drm_device *dev,
1643                                uint32_t flush_domains,
1644                                struct intel_ring_buffer *ring)
1645 {
1646         drm_i915_private_t *dev_priv = dev->dev_private;
1647         struct drm_i915_gem_object *obj_priv, *next;
1648
1649         list_for_each_entry_safe(obj_priv, next,
1650                                  &ring->gpu_write_list,
1651                                  gpu_write_list) {
1652                 struct drm_gem_object *obj = &obj_priv->base;
1653
1654                 if (obj->write_domain & flush_domains) {
1655                         uint32_t old_write_domain = obj->write_domain;
1656
1657                         obj->write_domain = 0;
1658                         list_del_init(&obj_priv->gpu_write_list);
1659                         i915_gem_object_move_to_active(obj, ring);
1660
1661                         /* update the fence lru list */
1662                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1663                                 struct drm_i915_fence_reg *reg =
1664                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1665                                 list_move_tail(&reg->lru_list,
1666                                                 &dev_priv->mm.fence_list);
1667                         }
1668
1669                         trace_i915_gem_object_change_domain(obj,
1670                                                             obj->read_domains,
1671                                                             old_write_domain);
1672                 }
1673         }
1674 }
1675
1676 uint32_t
1677 i915_add_request(struct drm_device *dev,
1678                  struct drm_file *file,
1679                  struct drm_i915_gem_request *request,
1680                  struct intel_ring_buffer *ring)
1681 {
1682         drm_i915_private_t *dev_priv = dev->dev_private;
1683         struct drm_i915_file_private *file_priv = NULL;
1684         uint32_t seqno;
1685         int was_empty;
1686
1687         if (file != NULL)
1688                 file_priv = file->driver_priv;
1689
1690         if (request == NULL) {
1691                 request = kzalloc(sizeof(*request), GFP_KERNEL);
1692                 if (request == NULL)
1693                         return 0;
1694         }
1695
1696         seqno = ring->add_request(dev, ring, 0);
1697         ring->outstanding_lazy_request = false;
1698
1699         request->seqno = seqno;
1700         request->ring = ring;
1701         request->emitted_jiffies = jiffies;
1702         was_empty = list_empty(&ring->request_list);
1703         list_add_tail(&request->list, &ring->request_list);
1704
1705         if (file_priv) {
1706                 spin_lock(&file_priv->mm.lock);
1707                 request->file_priv = file_priv;
1708                 list_add_tail(&request->client_list,
1709                               &file_priv->mm.request_list);
1710                 spin_unlock(&file_priv->mm.lock);
1711         }
1712
1713         if (!dev_priv->mm.suspended) {
1714                 mod_timer(&dev_priv->hangcheck_timer,
1715                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1716                 if (was_empty)
1717                         queue_delayed_work(dev_priv->wq,
1718                                            &dev_priv->mm.retire_work, HZ);
1719         }
1720         return seqno;
1721 }
1722
1723 /**
1724  * Command execution barrier
1725  *
1726  * Ensures that all commands in the ring are finished
1727  * before signalling the CPU
1728  */
1729 static void
1730 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1731 {
1732         uint32_t flush_domains = 0;
1733
1734         /* The sampler always gets flushed on i965 (sigh) */
1735         if (INTEL_INFO(dev)->gen >= 4)
1736                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1737
1738         ring->flush(dev, ring,
1739                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1740 }
1741
1742 static inline void
1743 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1744 {
1745         struct drm_i915_file_private *file_priv = request->file_priv;
1746
1747         if (!file_priv)
1748                 return;
1749
1750         spin_lock(&file_priv->mm.lock);
1751         list_del(&request->client_list);
1752         request->file_priv = NULL;
1753         spin_unlock(&file_priv->mm.lock);
1754 }
1755
1756 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1757                                       struct intel_ring_buffer *ring)
1758 {
1759         while (!list_empty(&ring->request_list)) {
1760                 struct drm_i915_gem_request *request;
1761
1762                 request = list_first_entry(&ring->request_list,
1763                                            struct drm_i915_gem_request,
1764                                            list);
1765
1766                 list_del(&request->list);
1767                 i915_gem_request_remove_from_client(request);
1768                 kfree(request);
1769         }
1770
1771         while (!list_empty(&ring->active_list)) {
1772                 struct drm_i915_gem_object *obj_priv;
1773
1774                 obj_priv = list_first_entry(&ring->active_list,
1775                                             struct drm_i915_gem_object,
1776                                             ring_list);
1777
1778                 obj_priv->base.write_domain = 0;
1779                 list_del_init(&obj_priv->gpu_write_list);
1780                 i915_gem_object_move_to_inactive(&obj_priv->base);
1781         }
1782 }
1783
1784 void i915_gem_reset(struct drm_device *dev)
1785 {
1786         struct drm_i915_private *dev_priv = dev->dev_private;
1787         struct drm_i915_gem_object *obj_priv;
1788         int i;
1789
1790         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1791         i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1792         i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1793
1794         /* Remove anything from the flushing lists. The GPU cache is likely
1795          * to be lost on reset along with the data, so simply move the
1796          * lost bo to the inactive list.
1797          */
1798         while (!list_empty(&dev_priv->mm.flushing_list)) {
1799                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1800                                             struct drm_i915_gem_object,
1801                                             mm_list);
1802
1803                 obj_priv->base.write_domain = 0;
1804                 list_del_init(&obj_priv->gpu_write_list);
1805                 i915_gem_object_move_to_inactive(&obj_priv->base);
1806         }
1807
1808         /* Move everything out of the GPU domains to ensure we do any
1809          * necessary invalidation upon reuse.
1810          */
1811         list_for_each_entry(obj_priv,
1812                             &dev_priv->mm.inactive_list,
1813                             mm_list)
1814         {
1815                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1816         }
1817
1818         /* The fence registers are invalidated so clear them out */
1819         for (i = 0; i < 16; i++) {
1820                 struct drm_i915_fence_reg *reg;
1821
1822                 reg = &dev_priv->fence_regs[i];
1823                 if (!reg->obj)
1824                         continue;
1825
1826                 i915_gem_clear_fence_reg(reg->obj);
1827         }
1828 }
1829
1830 /**
1831  * This function clears the request list as sequence numbers are passed.
1832  */
1833 static void
1834 i915_gem_retire_requests_ring(struct drm_device *dev,
1835                               struct intel_ring_buffer *ring)
1836 {
1837         drm_i915_private_t *dev_priv = dev->dev_private;
1838         uint32_t seqno;
1839
1840         if (!ring->status_page.page_addr ||
1841             list_empty(&ring->request_list))
1842                 return;
1843
1844         WARN_ON(i915_verify_lists(dev));
1845
1846         seqno = ring->get_seqno(dev, ring);
1847         while (!list_empty(&ring->request_list)) {
1848                 struct drm_i915_gem_request *request;
1849
1850                 request = list_first_entry(&ring->request_list,
1851                                            struct drm_i915_gem_request,
1852                                            list);
1853
1854                 if (!i915_seqno_passed(seqno, request->seqno))
1855                         break;
1856
1857                 trace_i915_gem_request_retire(dev, request->seqno);
1858
1859                 list_del(&request->list);
1860                 i915_gem_request_remove_from_client(request);
1861                 kfree(request);
1862         }
1863
1864         /* Move any buffers on the active list that are no longer referenced
1865          * by the ringbuffer to the flushing/inactive lists as appropriate.
1866          */
1867         while (!list_empty(&ring->active_list)) {
1868                 struct drm_gem_object *obj;
1869                 struct drm_i915_gem_object *obj_priv;
1870
1871                 obj_priv = list_first_entry(&ring->active_list,
1872                                             struct drm_i915_gem_object,
1873                                             ring_list);
1874
1875                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1876                         break;
1877
1878                 obj = &obj_priv->base;
1879                 if (obj->write_domain != 0)
1880                         i915_gem_object_move_to_flushing(obj);
1881                 else
1882                         i915_gem_object_move_to_inactive(obj);
1883         }
1884
1885         if (unlikely (dev_priv->trace_irq_seqno &&
1886                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1887                 ring->user_irq_put(dev, ring);
1888                 dev_priv->trace_irq_seqno = 0;
1889         }
1890
1891         WARN_ON(i915_verify_lists(dev));
1892 }
1893
1894 void
1895 i915_gem_retire_requests(struct drm_device *dev)
1896 {
1897         drm_i915_private_t *dev_priv = dev->dev_private;
1898
1899         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1900             struct drm_i915_gem_object *obj_priv, *tmp;
1901
1902             /* We must be careful that during unbind() we do not
1903              * accidentally infinitely recurse into retire requests.
1904              * Currently:
1905              *   retire -> free -> unbind -> wait -> retire_ring
1906              */
1907             list_for_each_entry_safe(obj_priv, tmp,
1908                                      &dev_priv->mm.deferred_free_list,
1909                                      mm_list)
1910                     i915_gem_free_object_tail(&obj_priv->base);
1911         }
1912
1913         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1914         i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1915         i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
1916 }
1917
1918 static void
1919 i915_gem_retire_work_handler(struct work_struct *work)
1920 {
1921         drm_i915_private_t *dev_priv;
1922         struct drm_device *dev;
1923
1924         dev_priv = container_of(work, drm_i915_private_t,
1925                                 mm.retire_work.work);
1926         dev = dev_priv->dev;
1927
1928         /* Come back later if the device is busy... */
1929         if (!mutex_trylock(&dev->struct_mutex)) {
1930                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1931                 return;
1932         }
1933
1934         i915_gem_retire_requests(dev);
1935
1936         if (!dev_priv->mm.suspended &&
1937                 (!list_empty(&dev_priv->render_ring.request_list) ||
1938                  !list_empty(&dev_priv->bsd_ring.request_list) ||
1939                  !list_empty(&dev_priv->blt_ring.request_list)))
1940                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1941         mutex_unlock(&dev->struct_mutex);
1942 }
1943
1944 int
1945 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1946                      bool interruptible, struct intel_ring_buffer *ring)
1947 {
1948         drm_i915_private_t *dev_priv = dev->dev_private;
1949         u32 ier;
1950         int ret = 0;
1951
1952         BUG_ON(seqno == 0);
1953
1954         if (atomic_read(&dev_priv->mm.wedged))
1955                 return -EAGAIN;
1956
1957         if (ring->outstanding_lazy_request) {
1958                 seqno = i915_add_request(dev, NULL, NULL, ring);
1959                 if (seqno == 0)
1960                         return -ENOMEM;
1961         }
1962         BUG_ON(seqno == dev_priv->next_seqno);
1963
1964         if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
1965                 if (HAS_PCH_SPLIT(dev))
1966                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1967                 else
1968                         ier = I915_READ(IER);
1969                 if (!ier) {
1970                         DRM_ERROR("something (likely vbetool) disabled "
1971                                   "interrupts, re-enabling\n");
1972                         i915_driver_irq_preinstall(dev);
1973                         i915_driver_irq_postinstall(dev);
1974                 }
1975
1976                 trace_i915_gem_request_wait_begin(dev, seqno);
1977
1978                 ring->waiting_gem_seqno = seqno;
1979                 ring->user_irq_get(dev, ring);
1980                 if (interruptible)
1981                         ret = wait_event_interruptible(ring->irq_queue,
1982                                 i915_seqno_passed(
1983                                         ring->get_seqno(dev, ring), seqno)
1984                                 || atomic_read(&dev_priv->mm.wedged));
1985                 else
1986                         wait_event(ring->irq_queue,
1987                                 i915_seqno_passed(
1988                                         ring->get_seqno(dev, ring), seqno)
1989                                 || atomic_read(&dev_priv->mm.wedged));
1990
1991                 ring->user_irq_put(dev, ring);
1992                 ring->waiting_gem_seqno = 0;
1993
1994                 trace_i915_gem_request_wait_end(dev, seqno);
1995         }
1996         if (atomic_read(&dev_priv->mm.wedged))
1997                 ret = -EAGAIN;
1998
1999         if (ret && ret != -ERESTARTSYS)
2000                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2001                           __func__, ret, seqno, ring->get_seqno(dev, ring),
2002                           dev_priv->next_seqno);
2003
2004         /* Directly dispatch request retiring.  While we have the work queue
2005          * to handle this, the waiter on a request often wants an associated
2006          * buffer to have made it to the inactive list, and we would need
2007          * a separate wait queue to handle that.
2008          */
2009         if (ret == 0)
2010                 i915_gem_retire_requests_ring(dev, ring);
2011
2012         return ret;
2013 }
2014
2015 /**
2016  * Waits for a sequence number to be signaled, and cleans up the
2017  * request and object lists appropriately for that event.
2018  */
2019 static int
2020 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2021                   struct intel_ring_buffer *ring)
2022 {
2023         return i915_do_wait_request(dev, seqno, 1, ring);
2024 }
2025
2026 static void
2027 i915_gem_flush_ring(struct drm_device *dev,
2028                     struct drm_file *file_priv,
2029                     struct intel_ring_buffer *ring,
2030                     uint32_t invalidate_domains,
2031                     uint32_t flush_domains)
2032 {
2033         ring->flush(dev, ring, invalidate_domains, flush_domains);
2034         i915_gem_process_flushing_list(dev, flush_domains, ring);
2035 }
2036
2037 static void
2038 i915_gem_flush(struct drm_device *dev,
2039                struct drm_file *file_priv,
2040                uint32_t invalidate_domains,
2041                uint32_t flush_domains,
2042                uint32_t flush_rings)
2043 {
2044         drm_i915_private_t *dev_priv = dev->dev_private;
2045
2046         if (flush_domains & I915_GEM_DOMAIN_CPU)
2047                 drm_agp_chipset_flush(dev);
2048
2049         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2050                 if (flush_rings & RING_RENDER)
2051                         i915_gem_flush_ring(dev, file_priv,
2052                                             &dev_priv->render_ring,
2053                                             invalidate_domains, flush_domains);
2054                 if (flush_rings & RING_BSD)
2055                         i915_gem_flush_ring(dev, file_priv,
2056                                             &dev_priv->bsd_ring,
2057                                             invalidate_domains, flush_domains);
2058                 if (flush_rings & RING_BLT)
2059                         i915_gem_flush_ring(dev, file_priv,
2060                                             &dev_priv->blt_ring,
2061                                             invalidate_domains, flush_domains);
2062         }
2063 }
2064
2065 /**
2066  * Ensures that all rendering to the object has completed and the object is
2067  * safe to unbind from the GTT or access from the CPU.
2068  */
2069 static int
2070 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2071                                bool interruptible)
2072 {
2073         struct drm_device *dev = obj->dev;
2074         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2075         int ret;
2076
2077         /* This function only exists to support waiting for existing rendering,
2078          * not for emitting required flushes.
2079          */
2080         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2081
2082         /* If there is rendering queued on the buffer being evicted, wait for
2083          * it.
2084          */
2085         if (obj_priv->active) {
2086                 ret = i915_do_wait_request(dev,
2087                                            obj_priv->last_rendering_seqno,
2088                                            interruptible,
2089                                            obj_priv->ring);
2090                 if (ret)
2091                         return ret;
2092         }
2093
2094         return 0;
2095 }
2096
2097 /**
2098  * Unbinds an object from the GTT aperture.
2099  */
2100 int
2101 i915_gem_object_unbind(struct drm_gem_object *obj)
2102 {
2103         struct drm_device *dev = obj->dev;
2104         struct drm_i915_private *dev_priv = dev->dev_private;
2105         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2106         int ret = 0;
2107
2108         if (obj_priv->gtt_space == NULL)
2109                 return 0;
2110
2111         if (obj_priv->pin_count != 0) {
2112                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2113                 return -EINVAL;
2114         }
2115
2116         /* blow away mappings if mapped through GTT */
2117         i915_gem_release_mmap(obj);
2118
2119         /* Move the object to the CPU domain to ensure that
2120          * any possible CPU writes while it's not in the GTT
2121          * are flushed when we go to remap it. This will
2122          * also ensure that all pending GPU writes are finished
2123          * before we unbind.
2124          */
2125         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2126         if (ret == -ERESTARTSYS)
2127                 return ret;
2128         /* Continue on if we fail due to EIO, the GPU is hung so we
2129          * should be safe and we need to cleanup or else we might
2130          * cause memory corruption through use-after-free.
2131          */
2132         if (ret) {
2133                 i915_gem_clflush_object(obj);
2134                 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2135         }
2136
2137         /* release the fence reg _after_ flushing */
2138         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2139                 i915_gem_clear_fence_reg(obj);
2140
2141         drm_unbind_agp(obj_priv->agp_mem);
2142         drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2143
2144         i915_gem_object_put_pages(obj);
2145         BUG_ON(obj_priv->pages_refcount);
2146
2147         i915_gem_info_remove_gtt(dev_priv, obj->size);
2148         list_del_init(&obj_priv->mm_list);
2149
2150         drm_mm_put_block(obj_priv->gtt_space);
2151         obj_priv->gtt_space = NULL;
2152         obj_priv->gtt_offset = 0;
2153
2154         if (i915_gem_object_is_purgeable(obj_priv))
2155                 i915_gem_object_truncate(obj);
2156
2157         trace_i915_gem_object_unbind(obj);
2158
2159         return ret;
2160 }
2161
2162 static int i915_ring_idle(struct drm_device *dev,
2163                           struct intel_ring_buffer *ring)
2164 {
2165         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2166                 return 0;
2167
2168         i915_gem_flush_ring(dev, NULL, ring,
2169                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2170         return i915_wait_request(dev,
2171                                  i915_gem_next_request_seqno(dev, ring),
2172                                  ring);
2173 }
2174
2175 int
2176 i915_gpu_idle(struct drm_device *dev)
2177 {
2178         drm_i915_private_t *dev_priv = dev->dev_private;
2179         bool lists_empty;
2180         int ret;
2181
2182         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2183                        list_empty(&dev_priv->mm.active_list));
2184         if (lists_empty)
2185                 return 0;
2186
2187         /* Flush everything onto the inactive list. */
2188         ret = i915_ring_idle(dev, &dev_priv->render_ring);
2189         if (ret)
2190                 return ret;
2191
2192         ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2193         if (ret)
2194                 return ret;
2195
2196         ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2197         if (ret)
2198                 return ret;
2199
2200         return 0;
2201 }
2202
2203 static int
2204 i915_gem_object_get_pages(struct drm_gem_object *obj,
2205                           gfp_t gfpmask)
2206 {
2207         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2208         int page_count, i;
2209         struct address_space *mapping;
2210         struct inode *inode;
2211         struct page *page;
2212
2213         BUG_ON(obj_priv->pages_refcount
2214                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2215
2216         if (obj_priv->pages_refcount++ != 0)
2217                 return 0;
2218
2219         /* Get the list of pages out of our struct file.  They'll be pinned
2220          * at this point until we release them.
2221          */
2222         page_count = obj->size / PAGE_SIZE;
2223         BUG_ON(obj_priv->pages != NULL);
2224         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2225         if (obj_priv->pages == NULL) {
2226                 obj_priv->pages_refcount--;
2227                 return -ENOMEM;
2228         }
2229
2230         inode = obj->filp->f_path.dentry->d_inode;
2231         mapping = inode->i_mapping;
2232         for (i = 0; i < page_count; i++) {
2233                 page = read_cache_page_gfp(mapping, i,
2234                                            GFP_HIGHUSER |
2235                                            __GFP_COLD |
2236                                            __GFP_RECLAIMABLE |
2237                                            gfpmask);
2238                 if (IS_ERR(page))
2239                         goto err_pages;
2240
2241                 obj_priv->pages[i] = page;
2242         }
2243
2244         if (obj_priv->tiling_mode != I915_TILING_NONE)
2245                 i915_gem_object_do_bit_17_swizzle(obj);
2246
2247         return 0;
2248
2249 err_pages:
2250         while (i--)
2251                 page_cache_release(obj_priv->pages[i]);
2252
2253         drm_free_large(obj_priv->pages);
2254         obj_priv->pages = NULL;
2255         obj_priv->pages_refcount--;
2256         return PTR_ERR(page);
2257 }
2258
2259 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2260 {
2261         struct drm_gem_object *obj = reg->obj;
2262         struct drm_device *dev = obj->dev;
2263         drm_i915_private_t *dev_priv = dev->dev_private;
2264         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2265         int regnum = obj_priv->fence_reg;
2266         uint64_t val;
2267
2268         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2269                     0xfffff000) << 32;
2270         val |= obj_priv->gtt_offset & 0xfffff000;
2271         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2272                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2273
2274         if (obj_priv->tiling_mode == I915_TILING_Y)
2275                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2276         val |= I965_FENCE_REG_VALID;
2277
2278         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2279 }
2280
2281 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2282 {
2283         struct drm_gem_object *obj = reg->obj;
2284         struct drm_device *dev = obj->dev;
2285         drm_i915_private_t *dev_priv = dev->dev_private;
2286         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2287         int regnum = obj_priv->fence_reg;
2288         uint64_t val;
2289
2290         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2291                     0xfffff000) << 32;
2292         val |= obj_priv->gtt_offset & 0xfffff000;
2293         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2294         if (obj_priv->tiling_mode == I915_TILING_Y)
2295                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2296         val |= I965_FENCE_REG_VALID;
2297
2298         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2299 }
2300
2301 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2302 {
2303         struct drm_gem_object *obj = reg->obj;
2304         struct drm_device *dev = obj->dev;
2305         drm_i915_private_t *dev_priv = dev->dev_private;
2306         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2307         int regnum = obj_priv->fence_reg;
2308         int tile_width;
2309         uint32_t fence_reg, val;
2310         uint32_t pitch_val;
2311
2312         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2313             (obj_priv->gtt_offset & (obj->size - 1))) {
2314                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2315                      __func__, obj_priv->gtt_offset, obj->size);
2316                 return;
2317         }
2318
2319         if (obj_priv->tiling_mode == I915_TILING_Y &&
2320             HAS_128_BYTE_Y_TILING(dev))
2321                 tile_width = 128;
2322         else
2323                 tile_width = 512;
2324
2325         /* Note: pitch better be a power of two tile widths */
2326         pitch_val = obj_priv->stride / tile_width;
2327         pitch_val = ffs(pitch_val) - 1;
2328
2329         if (obj_priv->tiling_mode == I915_TILING_Y &&
2330             HAS_128_BYTE_Y_TILING(dev))
2331                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2332         else
2333                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2334
2335         val = obj_priv->gtt_offset;
2336         if (obj_priv->tiling_mode == I915_TILING_Y)
2337                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2338         val |= I915_FENCE_SIZE_BITS(obj->size);
2339         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2340         val |= I830_FENCE_REG_VALID;
2341
2342         if (regnum < 8)
2343                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2344         else
2345                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2346         I915_WRITE(fence_reg, val);
2347 }
2348
2349 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2350 {
2351         struct drm_gem_object *obj = reg->obj;
2352         struct drm_device *dev = obj->dev;
2353         drm_i915_private_t *dev_priv = dev->dev_private;
2354         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2355         int regnum = obj_priv->fence_reg;
2356         uint32_t val;
2357         uint32_t pitch_val;
2358         uint32_t fence_size_bits;
2359
2360         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2361             (obj_priv->gtt_offset & (obj->size - 1))) {
2362                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2363                      __func__, obj_priv->gtt_offset);
2364                 return;
2365         }
2366
2367         pitch_val = obj_priv->stride / 128;
2368         pitch_val = ffs(pitch_val) - 1;
2369         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2370
2371         val = obj_priv->gtt_offset;
2372         if (obj_priv->tiling_mode == I915_TILING_Y)
2373                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2374         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2375         WARN_ON(fence_size_bits & ~0x00000f00);
2376         val |= fence_size_bits;
2377         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2378         val |= I830_FENCE_REG_VALID;
2379
2380         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2381 }
2382
2383 static int i915_find_fence_reg(struct drm_device *dev,
2384                                bool interruptible)
2385 {
2386         struct drm_i915_fence_reg *reg = NULL;
2387         struct drm_i915_gem_object *obj_priv = NULL;
2388         struct drm_i915_private *dev_priv = dev->dev_private;
2389         struct drm_gem_object *obj = NULL;
2390         int i, avail, ret;
2391
2392         /* First try to find a free reg */
2393         avail = 0;
2394         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2395                 reg = &dev_priv->fence_regs[i];
2396                 if (!reg->obj)
2397                         return i;
2398
2399                 obj_priv = to_intel_bo(reg->obj);
2400                 if (!obj_priv->pin_count)
2401                     avail++;
2402         }
2403
2404         if (avail == 0)
2405                 return -ENOSPC;
2406
2407         /* None available, try to steal one or wait for a user to finish */
2408         i = I915_FENCE_REG_NONE;
2409         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2410                             lru_list) {
2411                 obj = reg->obj;
2412                 obj_priv = to_intel_bo(obj);
2413
2414                 if (obj_priv->pin_count)
2415                         continue;
2416
2417                 /* found one! */
2418                 i = obj_priv->fence_reg;
2419                 break;
2420         }
2421
2422         BUG_ON(i == I915_FENCE_REG_NONE);
2423
2424         /* We only have a reference on obj from the active list. put_fence_reg
2425          * might drop that one, causing a use-after-free in it. So hold a
2426          * private reference to obj like the other callers of put_fence_reg
2427          * (set_tiling ioctl) do. */
2428         drm_gem_object_reference(obj);
2429         ret = i915_gem_object_put_fence_reg(obj, interruptible);
2430         drm_gem_object_unreference(obj);
2431         if (ret != 0)
2432                 return ret;
2433
2434         return i;
2435 }
2436
2437 /**
2438  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2439  * @obj: object to map through a fence reg
2440  *
2441  * When mapping objects through the GTT, userspace wants to be able to write
2442  * to them without having to worry about swizzling if the object is tiled.
2443  *
2444  * This function walks the fence regs looking for a free one for @obj,
2445  * stealing one if it can't find any.
2446  *
2447  * It then sets up the reg based on the object's properties: address, pitch
2448  * and tiling format.
2449  */
2450 int
2451 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2452                               bool interruptible)
2453 {
2454         struct drm_device *dev = obj->dev;
2455         struct drm_i915_private *dev_priv = dev->dev_private;
2456         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2457         struct drm_i915_fence_reg *reg = NULL;
2458         int ret;
2459
2460         /* Just update our place in the LRU if our fence is getting used. */
2461         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2462                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2463                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2464                 return 0;
2465         }
2466
2467         switch (obj_priv->tiling_mode) {
2468         case I915_TILING_NONE:
2469                 WARN(1, "allocating a fence for non-tiled object?\n");
2470                 break;
2471         case I915_TILING_X:
2472                 if (!obj_priv->stride)
2473                         return -EINVAL;
2474                 WARN((obj_priv->stride & (512 - 1)),
2475                      "object 0x%08x is X tiled but has non-512B pitch\n",
2476                      obj_priv->gtt_offset);
2477                 break;
2478         case I915_TILING_Y:
2479                 if (!obj_priv->stride)
2480                         return -EINVAL;
2481                 WARN((obj_priv->stride & (128 - 1)),
2482                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2483                      obj_priv->gtt_offset);
2484                 break;
2485         }
2486
2487         ret = i915_find_fence_reg(dev, interruptible);
2488         if (ret < 0)
2489                 return ret;
2490
2491         obj_priv->fence_reg = ret;
2492         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2493         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2494
2495         reg->obj = obj;
2496
2497         switch (INTEL_INFO(dev)->gen) {
2498         case 6:
2499                 sandybridge_write_fence_reg(reg);
2500                 break;
2501         case 5:
2502         case 4:
2503                 i965_write_fence_reg(reg);
2504                 break;
2505         case 3:
2506                 i915_write_fence_reg(reg);
2507                 break;
2508         case 2:
2509                 i830_write_fence_reg(reg);
2510                 break;
2511         }
2512
2513         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2514                         obj_priv->tiling_mode);
2515
2516         return 0;
2517 }
2518
2519 /**
2520  * i915_gem_clear_fence_reg - clear out fence register info
2521  * @obj: object to clear
2522  *
2523  * Zeroes out the fence register itself and clears out the associated
2524  * data structures in dev_priv and obj_priv.
2525  */
2526 static void
2527 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2528 {
2529         struct drm_device *dev = obj->dev;
2530         drm_i915_private_t *dev_priv = dev->dev_private;
2531         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2532         struct drm_i915_fence_reg *reg =
2533                 &dev_priv->fence_regs[obj_priv->fence_reg];
2534         uint32_t fence_reg;
2535
2536         switch (INTEL_INFO(dev)->gen) {
2537         case 6:
2538                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2539                              (obj_priv->fence_reg * 8), 0);
2540                 break;
2541         case 5:
2542         case 4:
2543                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2544                 break;
2545         case 3:
2546                 if (obj_priv->fence_reg >= 8)
2547                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2548                 else
2549         case 2:
2550                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2551
2552                 I915_WRITE(fence_reg, 0);
2553                 break;
2554         }
2555
2556         reg->obj = NULL;
2557         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2558         list_del_init(&reg->lru_list);
2559 }
2560
2561 /**
2562  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2563  * to the buffer to finish, and then resets the fence register.
2564  * @obj: tiled object holding a fence register.
2565  * @bool: whether the wait upon the fence is interruptible
2566  *
2567  * Zeroes out the fence register itself and clears out the associated
2568  * data structures in dev_priv and obj_priv.
2569  */
2570 int
2571 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2572                               bool interruptible)
2573 {
2574         struct drm_device *dev = obj->dev;
2575         struct drm_i915_private *dev_priv = dev->dev_private;
2576         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2577         struct drm_i915_fence_reg *reg;
2578
2579         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2580                 return 0;
2581
2582         /* If we've changed tiling, GTT-mappings of the object
2583          * need to re-fault to ensure that the correct fence register
2584          * setup is in place.
2585          */
2586         i915_gem_release_mmap(obj);
2587
2588         /* On the i915, GPU access to tiled buffers is via a fence,
2589          * therefore we must wait for any outstanding access to complete
2590          * before clearing the fence.
2591          */
2592         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2593         if (reg->gpu) {
2594                 int ret;
2595
2596                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2597                 if (ret)
2598                         return ret;
2599
2600                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2601                 if (ret)
2602                         return ret;
2603
2604                 reg->gpu = false;
2605         }
2606
2607         i915_gem_object_flush_gtt_write_domain(obj);
2608         i915_gem_clear_fence_reg(obj);
2609
2610         return 0;
2611 }
2612
2613 /**
2614  * Finds free space in the GTT aperture and binds the object there.
2615  */
2616 static int
2617 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2618 {
2619         struct drm_device *dev = obj->dev;
2620         drm_i915_private_t *dev_priv = dev->dev_private;
2621         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2622         struct drm_mm_node *free_space;
2623         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2624         int ret;
2625
2626         if (obj_priv->madv != I915_MADV_WILLNEED) {
2627                 DRM_ERROR("Attempting to bind a purgeable object\n");
2628                 return -EINVAL;
2629         }
2630
2631         if (alignment == 0)
2632                 alignment = i915_gem_get_gtt_alignment(obj);
2633         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2634                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2635                 return -EINVAL;
2636         }
2637
2638         /* If the object is bigger than the entire aperture, reject it early
2639          * before evicting everything in a vain attempt to find space.
2640          */
2641         if (obj->size > dev_priv->mm.gtt_total) {
2642                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2643                 return -E2BIG;
2644         }
2645
2646  search_free:
2647         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2648                                         obj->size, alignment, 0);
2649         if (free_space != NULL)
2650                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2651                                                        alignment);
2652         if (obj_priv->gtt_space == NULL) {
2653                 /* If the gtt is empty and we're still having trouble
2654                  * fitting our object in, we're out of memory.
2655                  */
2656                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2657                 if (ret)
2658                         return ret;
2659
2660                 goto search_free;
2661         }
2662
2663         ret = i915_gem_object_get_pages(obj, gfpmask);
2664         if (ret) {
2665                 drm_mm_put_block(obj_priv->gtt_space);
2666                 obj_priv->gtt_space = NULL;
2667
2668                 if (ret == -ENOMEM) {
2669                         /* first try to clear up some space from the GTT */
2670                         ret = i915_gem_evict_something(dev, obj->size,
2671                                                        alignment);
2672                         if (ret) {
2673                                 /* now try to shrink everyone else */
2674                                 if (gfpmask) {
2675                                         gfpmask = 0;
2676                                         goto search_free;
2677                                 }
2678
2679                                 return ret;
2680                         }
2681
2682                         goto search_free;
2683                 }
2684
2685                 return ret;
2686         }
2687
2688         /* Create an AGP memory structure pointing at our pages, and bind it
2689          * into the GTT.
2690          */
2691         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2692                                                obj_priv->pages,
2693                                                obj->size >> PAGE_SHIFT,
2694                                                obj_priv->gtt_space->start,
2695                                                obj_priv->agp_type);
2696         if (obj_priv->agp_mem == NULL) {
2697                 i915_gem_object_put_pages(obj);
2698                 drm_mm_put_block(obj_priv->gtt_space);
2699                 obj_priv->gtt_space = NULL;
2700
2701                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2702                 if (ret)
2703                         return ret;
2704
2705                 goto search_free;
2706         }
2707
2708         /* keep track of bounds object by adding it to the inactive list */
2709         list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
2710         i915_gem_info_add_gtt(dev_priv, obj->size);
2711
2712         /* Assert that the object is not currently in any GPU domain. As it
2713          * wasn't in the GTT, there shouldn't be any way it could have been in
2714          * a GPU cache
2715          */
2716         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2717         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2718
2719         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2720         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2721
2722         return 0;
2723 }
2724
2725 void
2726 i915_gem_clflush_object(struct drm_gem_object *obj)
2727 {
2728         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2729
2730         /* If we don't have a page list set up, then we're not pinned
2731          * to GPU, and we can ignore the cache flush because it'll happen
2732          * again at bind time.
2733          */
2734         if (obj_priv->pages == NULL)
2735                 return;
2736
2737         trace_i915_gem_object_clflush(obj);
2738
2739         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2740 }
2741
2742 /** Flushes any GPU write domain for the object if it's dirty. */
2743 static int
2744 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2745 {
2746         struct drm_device *dev = obj->dev;
2747         uint32_t old_write_domain;
2748
2749         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2750                 return 0;
2751
2752         /* Queue the GPU write cache flushing we need. */
2753         old_write_domain = obj->write_domain;
2754         i915_gem_flush_ring(dev, NULL,
2755                             to_intel_bo(obj)->ring,
2756                             0, obj->write_domain);
2757         BUG_ON(obj->write_domain);
2758
2759         trace_i915_gem_object_change_domain(obj,
2760                                             obj->read_domains,
2761                                             old_write_domain);
2762
2763         return 0;
2764 }
2765
2766 /** Flushes the GTT write domain for the object if it's dirty. */
2767 static void
2768 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2769 {
2770         uint32_t old_write_domain;
2771
2772         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2773                 return;
2774
2775         /* No actual flushing is required for the GTT write domain.   Writes
2776          * to it immediately go to main memory as far as we know, so there's
2777          * no chipset flush.  It also doesn't land in render cache.
2778          */
2779         old_write_domain = obj->write_domain;
2780         obj->write_domain = 0;
2781
2782         trace_i915_gem_object_change_domain(obj,
2783                                             obj->read_domains,
2784                                             old_write_domain);
2785 }
2786
2787 /** Flushes the CPU write domain for the object if it's dirty. */
2788 static void
2789 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2790 {
2791         struct drm_device *dev = obj->dev;
2792         uint32_t old_write_domain;
2793
2794         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2795                 return;
2796
2797         i915_gem_clflush_object(obj);
2798         drm_agp_chipset_flush(dev);
2799         old_write_domain = obj->write_domain;
2800         obj->write_domain = 0;
2801
2802         trace_i915_gem_object_change_domain(obj,
2803                                             obj->read_domains,
2804                                             old_write_domain);
2805 }
2806
2807 /**
2808  * Moves a single object to the GTT read, and possibly write domain.
2809  *
2810  * This function returns when the move is complete, including waiting on
2811  * flushes to occur.
2812  */
2813 int
2814 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2815 {
2816         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2817         uint32_t old_write_domain, old_read_domains;
2818         int ret;
2819
2820         /* Not valid to be called on unbound objects. */
2821         if (obj_priv->gtt_space == NULL)
2822                 return -EINVAL;
2823
2824         ret = i915_gem_object_flush_gpu_write_domain(obj);
2825         if (ret != 0)
2826                 return ret;
2827         ret = i915_gem_object_wait_rendering(obj, true);
2828         if (ret)
2829                 return ret;
2830
2831         i915_gem_object_flush_cpu_write_domain(obj);
2832
2833         old_write_domain = obj->write_domain;
2834         old_read_domains = obj->read_domains;
2835
2836         /* It should now be out of any other write domains, and we can update
2837          * the domain values for our changes.
2838          */
2839         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2840         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2841         if (write) {
2842                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2843                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2844                 obj_priv->dirty = 1;
2845         }
2846
2847         trace_i915_gem_object_change_domain(obj,
2848                                             old_read_domains,
2849                                             old_write_domain);
2850
2851         return 0;
2852 }
2853
2854 /*
2855  * Prepare buffer for display plane. Use uninterruptible for possible flush
2856  * wait, as in modesetting process we're not supposed to be interrupted.
2857  */
2858 int
2859 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2860                                      bool pipelined)
2861 {
2862         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2863         uint32_t old_read_domains;
2864         int ret;
2865
2866         /* Not valid to be called on unbound objects. */
2867         if (obj_priv->gtt_space == NULL)
2868                 return -EINVAL;
2869
2870         ret = i915_gem_object_flush_gpu_write_domain(obj);
2871         if (ret)
2872                 return ret;
2873
2874         /* Currently, we are always called from an non-interruptible context. */
2875         if (!pipelined) {
2876                 ret = i915_gem_object_wait_rendering(obj, false);
2877                 if (ret)
2878                         return ret;
2879         }
2880
2881         i915_gem_object_flush_cpu_write_domain(obj);
2882
2883         old_read_domains = obj->read_domains;
2884         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2885
2886         trace_i915_gem_object_change_domain(obj,
2887                                             old_read_domains,
2888                                             obj->write_domain);
2889
2890         return 0;
2891 }
2892
2893 int
2894 i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2895                           bool interruptible)
2896 {
2897         if (!obj->active)
2898                 return 0;
2899
2900         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2901                 i915_gem_flush_ring(obj->base.dev, NULL, obj->ring,
2902                                     0, obj->base.write_domain);
2903
2904         return i915_gem_object_wait_rendering(&obj->base, interruptible);
2905 }
2906
2907 /**
2908  * Moves a single object to the CPU read, and possibly write domain.
2909  *
2910  * This function returns when the move is complete, including waiting on
2911  * flushes to occur.
2912  */
2913 static int
2914 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2915 {
2916         uint32_t old_write_domain, old_read_domains;
2917         int ret;
2918
2919         ret = i915_gem_object_flush_gpu_write_domain(obj);
2920         if (ret != 0)
2921                 return ret;
2922         ret = i915_gem_object_wait_rendering(obj, true);
2923         if (ret)
2924                 return ret;
2925
2926         i915_gem_object_flush_gtt_write_domain(obj);
2927
2928         /* If we have a partially-valid cache of the object in the CPU,
2929          * finish invalidating it and free the per-page flags.
2930          */
2931         i915_gem_object_set_to_full_cpu_read_domain(obj);
2932
2933         old_write_domain = obj->write_domain;
2934         old_read_domains = obj->read_domains;
2935
2936         /* Flush the CPU cache if it's still invalid. */
2937         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2938                 i915_gem_clflush_object(obj);
2939
2940                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2941         }
2942
2943         /* It should now be out of any other write domains, and we can update
2944          * the domain values for our changes.
2945          */
2946         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2947
2948         /* If we're writing through the CPU, then the GPU read domains will
2949          * need to be invalidated at next use.
2950          */
2951         if (write) {
2952                 obj->read_domains = I915_GEM_DOMAIN_CPU;
2953                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2954         }
2955
2956         trace_i915_gem_object_change_domain(obj,
2957                                             old_read_domains,
2958                                             old_write_domain);
2959
2960         return 0;
2961 }
2962
2963 /*
2964  * Set the next domain for the specified object. This
2965  * may not actually perform the necessary flushing/invaliding though,
2966  * as that may want to be batched with other set_domain operations
2967  *
2968  * This is (we hope) the only really tricky part of gem. The goal
2969  * is fairly simple -- track which caches hold bits of the object
2970  * and make sure they remain coherent. A few concrete examples may
2971  * help to explain how it works. For shorthand, we use the notation
2972  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2973  * a pair of read and write domain masks.
2974  *
2975  * Case 1: the batch buffer
2976  *
2977  *      1. Allocated
2978  *      2. Written by CPU
2979  *      3. Mapped to GTT
2980  *      4. Read by GPU
2981  *      5. Unmapped from GTT
2982  *      6. Freed
2983  *
2984  *      Let's take these a step at a time
2985  *
2986  *      1. Allocated
2987  *              Pages allocated from the kernel may still have
2988  *              cache contents, so we set them to (CPU, CPU) always.
2989  *      2. Written by CPU (using pwrite)
2990  *              The pwrite function calls set_domain (CPU, CPU) and
2991  *              this function does nothing (as nothing changes)
2992  *      3. Mapped by GTT
2993  *              This function asserts that the object is not
2994  *              currently in any GPU-based read or write domains
2995  *      4. Read by GPU
2996  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2997  *              As write_domain is zero, this function adds in the
2998  *              current read domains (CPU+COMMAND, 0).
2999  *              flush_domains is set to CPU.
3000  *              invalidate_domains is set to COMMAND
3001  *              clflush is run to get data out of the CPU caches
3002  *              then i915_dev_set_domain calls i915_gem_flush to
3003  *              emit an MI_FLUSH and drm_agp_chipset_flush
3004  *      5. Unmapped from GTT
3005  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3006  *              flush_domains and invalidate_domains end up both zero
3007  *              so no flushing/invalidating happens
3008  *      6. Freed
3009  *              yay, done
3010  *
3011  * Case 2: The shared render buffer
3012  *
3013  *      1. Allocated
3014  *      2. Mapped to GTT
3015  *      3. Read/written by GPU
3016  *      4. set_domain to (CPU,CPU)
3017  *      5. Read/written by CPU
3018  *      6. Read/written by GPU
3019  *
3020  *      1. Allocated
3021  *              Same as last example, (CPU, CPU)
3022  *      2. Mapped to GTT
3023  *              Nothing changes (assertions find that it is not in the GPU)
3024  *      3. Read/written by GPU
3025  *              execbuffer calls set_domain (RENDER, RENDER)
3026  *              flush_domains gets CPU
3027  *              invalidate_domains gets GPU
3028  *              clflush (obj)
3029  *              MI_FLUSH and drm_agp_chipset_flush
3030  *      4. set_domain (CPU, CPU)
3031  *              flush_domains gets GPU
3032  *              invalidate_domains gets CPU
3033  *              wait_rendering (obj) to make sure all drawing is complete.
3034  *              This will include an MI_FLUSH to get the data from GPU
3035  *              to memory
3036  *              clflush (obj) to invalidate the CPU cache
3037  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3038  *      5. Read/written by CPU
3039  *              cache lines are loaded and dirtied
3040  *      6. Read written by GPU
3041  *              Same as last GPU access
3042  *
3043  * Case 3: The constant buffer
3044  *
3045  *      1. Allocated
3046  *      2. Written by CPU
3047  *      3. Read by GPU
3048  *      4. Updated (written) by CPU again
3049  *      5. Read by GPU
3050  *
3051  *      1. Allocated
3052  *              (CPU, CPU)
3053  *      2. Written by CPU
3054  *              (CPU, CPU)
3055  *      3. Read by GPU
3056  *              (CPU+RENDER, 0)
3057  *              flush_domains = CPU
3058  *              invalidate_domains = RENDER
3059  *              clflush (obj)
3060  *              MI_FLUSH
3061  *              drm_agp_chipset_flush
3062  *      4. Updated (written) by CPU again
3063  *              (CPU, CPU)
3064  *              flush_domains = 0 (no previous write domain)
3065  *              invalidate_domains = 0 (no new read domains)
3066  *      5. Read by GPU
3067  *              (CPU+RENDER, 0)
3068  *              flush_domains = CPU
3069  *              invalidate_domains = RENDER
3070  *              clflush (obj)
3071  *              MI_FLUSH
3072  *              drm_agp_chipset_flush
3073  */
3074 static void
3075 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3076                                   struct intel_ring_buffer *ring)
3077 {
3078         struct drm_device               *dev = obj->dev;
3079         struct drm_i915_private         *dev_priv = dev->dev_private;
3080         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3081         uint32_t                        invalidate_domains = 0;
3082         uint32_t                        flush_domains = 0;
3083         uint32_t                        old_read_domains;
3084
3085         intel_mark_busy(dev, obj);
3086
3087         /*
3088          * If the object isn't moving to a new write domain,
3089          * let the object stay in multiple read domains
3090          */
3091         if (obj->pending_write_domain == 0)
3092                 obj->pending_read_domains |= obj->read_domains;
3093         else
3094                 obj_priv->dirty = 1;
3095
3096         /*
3097          * Flush the current write domain if
3098          * the new read domains don't match. Invalidate
3099          * any read domains which differ from the old
3100          * write domain
3101          */
3102         if (obj->write_domain &&
3103             (obj->write_domain != obj->pending_read_domains ||
3104              obj_priv->ring != ring)) {
3105                 flush_domains |= obj->write_domain;
3106                 invalidate_domains |=
3107                         obj->pending_read_domains & ~obj->write_domain;
3108         }
3109         /*
3110          * Invalidate any read caches which may have
3111          * stale data. That is, any new read domains.
3112          */
3113         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3114         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3115                 i915_gem_clflush_object(obj);
3116
3117         old_read_domains = obj->read_domains;
3118
3119         /* The actual obj->write_domain will be updated with
3120          * pending_write_domain after we emit the accumulated flush for all
3121          * of our domain changes in execbuffers (which clears objects'
3122          * write_domains).  So if we have a current write domain that we
3123          * aren't changing, set pending_write_domain to that.
3124          */
3125         if (flush_domains == 0 && obj->pending_write_domain == 0)
3126                 obj->pending_write_domain = obj->write_domain;
3127         obj->read_domains = obj->pending_read_domains;
3128
3129         dev->invalidate_domains |= invalidate_domains;
3130         dev->flush_domains |= flush_domains;
3131         if (flush_domains & I915_GEM_GPU_DOMAINS)
3132                 dev_priv->mm.flush_rings |= obj_priv->ring->id;
3133         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3134                 dev_priv->mm.flush_rings |= ring->id;
3135
3136         trace_i915_gem_object_change_domain(obj,
3137                                             old_read_domains,
3138                                             obj->write_domain);
3139 }
3140
3141 /**
3142  * Moves the object from a partially CPU read to a full one.
3143  *
3144  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3145  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3146  */
3147 static void
3148 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3149 {
3150         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3151
3152         if (!obj_priv->page_cpu_valid)
3153                 return;
3154
3155         /* If we're partially in the CPU read domain, finish moving it in.
3156          */
3157         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3158                 int i;
3159
3160                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3161                         if (obj_priv->page_cpu_valid[i])
3162                                 continue;
3163                         drm_clflush_pages(obj_priv->pages + i, 1);
3164                 }
3165         }
3166
3167         /* Free the page_cpu_valid mappings which are now stale, whether
3168          * or not we've got I915_GEM_DOMAIN_CPU.
3169          */
3170         kfree(obj_priv->page_cpu_valid);
3171         obj_priv->page_cpu_valid = NULL;
3172 }
3173
3174 /**
3175  * Set the CPU read domain on a range of the object.
3176  *
3177  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3178  * not entirely valid.  The page_cpu_valid member of the object flags which
3179  * pages have been flushed, and will be respected by
3180  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3181  * of the whole object.
3182  *
3183  * This function returns when the move is complete, including waiting on
3184  * flushes to occur.
3185  */
3186 static int
3187 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3188                                           uint64_t offset, uint64_t size)
3189 {
3190         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3191         uint32_t old_read_domains;
3192         int i, ret;
3193
3194         if (offset == 0 && size == obj->size)
3195                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3196
3197         ret = i915_gem_object_flush_gpu_write_domain(obj);
3198         if (ret != 0)
3199                 return ret;
3200         ret = i915_gem_object_wait_rendering(obj, true);
3201         if (ret)
3202                 return ret;
3203
3204         i915_gem_object_flush_gtt_write_domain(obj);
3205
3206         /* If we're already fully in the CPU read domain, we're done. */
3207         if (obj_priv->page_cpu_valid == NULL &&
3208             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3209                 return 0;
3210
3211         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3212          * newly adding I915_GEM_DOMAIN_CPU
3213          */
3214         if (obj_priv->page_cpu_valid == NULL) {
3215                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3216                                                    GFP_KERNEL);
3217                 if (obj_priv->page_cpu_valid == NULL)
3218                         return -ENOMEM;
3219         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3220                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3221
3222         /* Flush the cache on any pages that are still invalid from the CPU's
3223          * perspective.
3224          */
3225         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3226              i++) {
3227                 if (obj_priv->page_cpu_valid[i])
3228                         continue;
3229
3230                 drm_clflush_pages(obj_priv->pages + i, 1);
3231
3232                 obj_priv->page_cpu_valid[i] = 1;
3233         }
3234
3235         /* It should now be out of any other write domains, and we can update
3236          * the domain values for our changes.
3237          */
3238         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3239
3240         old_read_domains = obj->read_domains;
3241         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3242
3243         trace_i915_gem_object_change_domain(obj,
3244                                             old_read_domains,
3245                                             obj->write_domain);
3246
3247         return 0;
3248 }
3249
3250 static int
3251 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
3252                                    struct drm_file *file_priv,
3253                                    struct drm_i915_gem_exec_object2 *entry,
3254                                    struct drm_i915_gem_relocation_entry *reloc)
3255 {
3256         struct drm_device *dev = obj->base.dev;
3257         struct drm_gem_object *target_obj;
3258         uint32_t target_offset;
3259         int ret = -EINVAL;
3260
3261         target_obj = drm_gem_object_lookup(dev, file_priv,
3262                                            reloc->target_handle);
3263         if (target_obj == NULL)
3264                 return -ENOENT;
3265
3266         target_offset = to_intel_bo(target_obj)->gtt_offset;
3267
3268 #if WATCH_RELOC
3269         DRM_INFO("%s: obj %p offset %08x target %d "
3270                  "read %08x write %08x gtt %08x "
3271                  "presumed %08x delta %08x\n",
3272                  __func__,
3273                  obj,
3274                  (int) reloc->offset,
3275                  (int) reloc->target_handle,
3276                  (int) reloc->read_domains,
3277                  (int) reloc->write_domain,
3278                  (int) target_offset,
3279                  (int) reloc->presumed_offset,
3280                  reloc->delta);
3281 #endif
3282
3283         /* The target buffer should have appeared before us in the
3284          * exec_object list, so it should have a GTT space bound by now.
3285          */
3286         if (target_offset == 0) {
3287                 DRM_ERROR("No GTT space found for object %d\n",
3288                           reloc->target_handle);
3289                 goto err;
3290         }
3291
3292         /* Validate that the target is in a valid r/w GPU domain */
3293         if (reloc->write_domain & (reloc->write_domain - 1)) {
3294                 DRM_ERROR("reloc with multiple write domains: "
3295                           "obj %p target %d offset %d "
3296                           "read %08x write %08x",
3297                           obj, reloc->target_handle,
3298                           (int) reloc->offset,
3299                           reloc->read_domains,
3300                           reloc->write_domain);
3301                 goto err;
3302         }
3303         if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3304             reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3305                 DRM_ERROR("reloc with read/write CPU domains: "
3306                           "obj %p target %d offset %d "
3307                           "read %08x write %08x",
3308                           obj, reloc->target_handle,
3309                           (int) reloc->offset,
3310                           reloc->read_domains,
3311                           reloc->write_domain);
3312                 goto err;
3313         }
3314         if (reloc->write_domain && target_obj->pending_write_domain &&
3315             reloc->write_domain != target_obj->pending_write_domain) {
3316                 DRM_ERROR("Write domain conflict: "
3317                           "obj %p target %d offset %d "
3318                           "new %08x old %08x\n",
3319                           obj, reloc->target_handle,
3320                           (int) reloc->offset,
3321                           reloc->write_domain,
3322                           target_obj->pending_write_domain);
3323                 goto err;
3324         }
3325
3326         target_obj->pending_read_domains |= reloc->read_domains;
3327         target_obj->pending_write_domain |= reloc->write_domain;
3328
3329         /* If the relocation already has the right value in it, no
3330          * more work needs to be done.
3331          */
3332         if (target_offset == reloc->presumed_offset)
3333                 goto out;
3334
3335         /* Check that the relocation address is valid... */
3336         if (reloc->offset > obj->base.size - 4) {
3337                 DRM_ERROR("Relocation beyond object bounds: "
3338                           "obj %p target %d offset %d size %d.\n",
3339                           obj, reloc->target_handle,
3340                           (int) reloc->offset,
3341                           (int) obj->base.size);
3342                 goto err;
3343         }
3344         if (reloc->offset & 3) {
3345                 DRM_ERROR("Relocation not 4-byte aligned: "
3346                           "obj %p target %d offset %d.\n",
3347                           obj, reloc->target_handle,
3348                           (int) reloc->offset);
3349                 goto err;
3350         }
3351
3352         /* and points to somewhere within the target object. */
3353         if (reloc->delta >= target_obj->size) {
3354                 DRM_ERROR("Relocation beyond target object bounds: "
3355                           "obj %p target %d delta %d size %d.\n",
3356                           obj, reloc->target_handle,
3357                           (int) reloc->delta,
3358                           (int) target_obj->size);
3359                 goto err;
3360         }
3361
3362         reloc->delta += target_offset;
3363         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3364                 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
3365                 char *vaddr;
3366
3367                 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
3368                 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
3369                 kunmap_atomic(vaddr);
3370         } else {
3371                 struct drm_i915_private *dev_priv = dev->dev_private;
3372                 uint32_t __iomem *reloc_entry;
3373                 void __iomem *reloc_page;
3374
3375                 ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3376                 if (ret)
3377                         goto err;
3378
3379                 /* Map the page containing the relocation we're going to perform.  */
3380                 reloc->offset += obj->gtt_offset;
3381                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3382                                                       reloc->offset & PAGE_MASK);
3383                 reloc_entry = (uint32_t __iomem *)
3384                         (reloc_page + (reloc->offset & ~PAGE_MASK));
3385                 iowrite32(reloc->delta, reloc_entry);
3386                 io_mapping_unmap_atomic(reloc_page);
3387         }
3388
3389         /* and update the user's relocation entry */
3390         reloc->presumed_offset = target_offset;
3391
3392 out:
3393         ret = 0;
3394 err:
3395         drm_gem_object_unreference(target_obj);
3396         return ret;
3397 }
3398
3399 static int
3400 i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
3401                                     struct drm_file *file_priv,
3402                                     struct drm_i915_gem_exec_object2 *entry)
3403 {
3404         struct drm_i915_gem_relocation_entry __user *user_relocs;
3405         int i, ret;
3406
3407         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3408         for (i = 0; i < entry->relocation_count; i++) {
3409                 struct drm_i915_gem_relocation_entry reloc;
3410
3411                 if (__copy_from_user_inatomic(&reloc,
3412                                               user_relocs+i,
3413                                               sizeof(reloc)))
3414                         return -EFAULT;
3415
3416                 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &reloc);
3417                 if (ret)
3418                         return ret;
3419
3420                 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3421                                             &reloc.presumed_offset,
3422                                             sizeof(reloc.presumed_offset)))
3423                         return -EFAULT;
3424         }
3425
3426         return 0;
3427 }
3428
3429 static int
3430 i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
3431                                          struct drm_file *file_priv,
3432                                          struct drm_i915_gem_exec_object2 *entry,
3433                                          struct drm_i915_gem_relocation_entry *relocs)
3434 {
3435         int i, ret;
3436
3437         for (i = 0; i < entry->relocation_count; i++) {
3438                 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &relocs[i]);
3439                 if (ret)
3440                         return ret;
3441         }
3442
3443         return 0;
3444 }
3445
3446 static int
3447 i915_gem_execbuffer_relocate(struct drm_device *dev,
3448                              struct drm_file *file,
3449                              struct drm_gem_object **object_list,
3450                              struct drm_i915_gem_exec_object2 *exec_list,
3451                              int count)
3452 {
3453         int i, ret;
3454
3455         for (i = 0; i < count; i++) {
3456                 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3457                 obj->base.pending_read_domains = 0;
3458                 obj->base.pending_write_domain = 0;
3459                 ret = i915_gem_execbuffer_relocate_object(obj, file,
3460                                                           &exec_list[i]);
3461                 if (ret)
3462                         return ret;
3463         }
3464
3465         return 0;
3466 }
3467
3468 static int
3469 i915_gem_execbuffer_reserve(struct drm_device *dev,
3470                             struct drm_file *file,
3471                             struct drm_gem_object **object_list,
3472                             struct drm_i915_gem_exec_object2 *exec_list,
3473                             int count)
3474 {
3475         struct drm_i915_private *dev_priv = dev->dev_private;
3476         int ret, i, retry;
3477
3478         /* attempt to pin all of the buffers into the GTT */
3479         for (retry = 0; retry < 2; retry++) {
3480                 ret = 0;
3481                 for (i = 0; i < count; i++) {
3482                         struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3483                         struct drm_i915_gem_object *obj= to_intel_bo(object_list[i]);
3484                         bool need_fence =
3485                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3486                                 obj->tiling_mode != I915_TILING_NONE;
3487
3488                         /* Check fence reg constraints and rebind if necessary */
3489                         if (need_fence &&
3490                             !i915_gem_object_fence_offset_ok(&obj->base,
3491                                                              obj->tiling_mode)) {
3492                                 ret = i915_gem_object_unbind(&obj->base);
3493                                 if (ret)
3494                                         break;
3495                         }
3496
3497                         ret = i915_gem_object_pin(&obj->base, entry->alignment);
3498                         if (ret)
3499                                 break;
3500
3501                         /*
3502                          * Pre-965 chips need a fence register set up in order
3503                          * to properly handle blits to/from tiled surfaces.
3504                          */
3505                         if (need_fence) {
3506                                 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3507                                 if (ret) {
3508                                         i915_gem_object_unpin(&obj->base);
3509                                         break;
3510                                 }
3511
3512                                 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3513                         }
3514
3515                         entry->offset = obj->gtt_offset;
3516                 }
3517
3518                 while (i--)
3519                         i915_gem_object_unpin(object_list[i]);
3520
3521                 if (ret == 0)
3522                         break;
3523
3524                 if (ret != -ENOSPC || retry)
3525                         return ret;
3526
3527                 ret = i915_gem_evict_everything(dev);
3528                 if (ret)
3529                         return ret;
3530         }
3531
3532         return 0;
3533 }
3534
3535 static int
3536 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3537                                   struct drm_file *file,
3538                                   struct drm_gem_object **object_list,
3539                                   struct drm_i915_gem_exec_object2 *exec_list,
3540                                   int count)
3541 {
3542         struct drm_i915_gem_relocation_entry *reloc;
3543         int i, total, ret;
3544
3545         for (i = 0; i < count; i++) {
3546                 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3547                 obj->in_execbuffer = false;
3548         }
3549
3550         mutex_unlock(&dev->struct_mutex);
3551
3552         total = 0;
3553         for (i = 0; i < count; i++)
3554                 total += exec_list[i].relocation_count;
3555
3556         reloc = drm_malloc_ab(total, sizeof(*reloc));
3557         if (reloc == NULL) {
3558                 mutex_lock(&dev->struct_mutex);
3559                 return -ENOMEM;
3560         }
3561
3562         total = 0;
3563         for (i = 0; i < count; i++) {
3564                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3565
3566                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3567
3568                 if (copy_from_user(reloc+total, user_relocs,
3569                                    exec_list[i].relocation_count *
3570                                    sizeof(*reloc))) {
3571                         ret = -EFAULT;
3572                         mutex_lock(&dev->struct_mutex);
3573                         goto err;
3574                 }
3575
3576                 total += exec_list[i].relocation_count;
3577         }
3578
3579         ret = i915_mutex_lock_interruptible(dev);
3580         if (ret) {
3581                 mutex_lock(&dev->struct_mutex);
3582                 goto err;
3583         }
3584
3585         ret = i915_gem_execbuffer_reserve(dev, file,
3586                                           object_list, exec_list,
3587                                           count);
3588         if (ret)
3589                 goto err;
3590
3591         total = 0;
3592         for (i = 0; i < count; i++) {
3593                 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3594                 obj->base.pending_read_domains = 0;
3595                 obj->base.pending_write_domain = 0;
3596                 ret = i915_gem_execbuffer_relocate_object_slow(obj, file,
3597                                                                &exec_list[i],
3598                                                                reloc + total);
3599                 if (ret)
3600                         goto err;
3601
3602                 total += exec_list[i].relocation_count;
3603         }
3604
3605         /* Leave the user relocations as are, this is the painfully slow path,
3606          * and we want to avoid the complication of dropping the lock whilst
3607          * having buffers reserved in the aperture and so causing spurious
3608          * ENOSPC for random operations.
3609          */
3610
3611 err:
3612         drm_free_large(reloc);
3613         return ret;
3614 }
3615
3616 static int
3617 i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3618                                 struct drm_file *file,
3619                                 struct intel_ring_buffer *ring,
3620                                 struct drm_gem_object **objects,
3621                                 int count)
3622 {
3623         struct drm_i915_private *dev_priv = dev->dev_private;
3624         int ret, i;
3625
3626         /* Zero the global flush/invalidate flags. These
3627          * will be modified as new domains are computed
3628          * for each object
3629          */
3630         dev->invalidate_domains = 0;
3631         dev->flush_domains = 0;
3632         dev_priv->mm.flush_rings = 0;
3633         for (i = 0; i < count; i++)
3634                 i915_gem_object_set_to_gpu_domain(objects[i], ring);
3635
3636         if (dev->invalidate_domains | dev->flush_domains) {
3637 #if WATCH_EXEC
3638                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3639                           __func__,
3640                          dev->invalidate_domains,
3641                          dev->flush_domains);
3642 #endif
3643                 i915_gem_flush(dev, file,
3644                                dev->invalidate_domains,
3645                                dev->flush_domains,
3646                                dev_priv->mm.flush_rings);
3647         }
3648
3649         for (i = 0; i < count; i++) {
3650                 struct drm_i915_gem_object *obj = to_intel_bo(objects[i]);
3651                 /* XXX replace with semaphores */
3652                 if (obj->ring && ring != obj->ring) {
3653                         ret = i915_gem_object_wait_rendering(&obj->base, true);
3654                         if (ret)
3655                                 return ret;
3656                 }
3657         }
3658
3659         return 0;
3660 }
3661
3662 /* Throttle our rendering by waiting until the ring has completed our requests
3663  * emitted over 20 msec ago.
3664  *
3665  * Note that if we were to use the current jiffies each time around the loop,
3666  * we wouldn't escape the function with any frames outstanding if the time to
3667  * render a frame was over 20ms.
3668  *
3669  * This should get us reasonable parallelism between CPU and GPU but also
3670  * relatively low latency when blocking on a particular request to finish.
3671  */
3672 static int
3673 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3674 {
3675         struct drm_i915_private *dev_priv = dev->dev_private;
3676         struct drm_i915_file_private *file_priv = file->driver_priv;
3677         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3678         struct drm_i915_gem_request *request;
3679         struct intel_ring_buffer *ring = NULL;
3680         u32 seqno = 0;
3681         int ret;
3682
3683         spin_lock(&file_priv->mm.lock);
3684         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3685                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3686                         break;
3687
3688                 ring = request->ring;
3689                 seqno = request->seqno;
3690         }
3691         spin_unlock(&file_priv->mm.lock);
3692
3693         if (seqno == 0)
3694                 return 0;
3695
3696         ret = 0;
3697         if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
3698                 /* And wait for the seqno passing without holding any locks and
3699                  * causing extra latency for others. This is safe as the irq
3700                  * generation is designed to be run atomically and so is
3701                  * lockless.
3702                  */
3703                 ring->user_irq_get(dev, ring);
3704                 ret = wait_event_interruptible(ring->irq_queue,
3705                                                i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
3706                                                || atomic_read(&dev_priv->mm.wedged));
3707                 ring->user_irq_put(dev, ring);
3708
3709                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3710                         ret = -EIO;
3711         }
3712
3713         if (ret == 0)
3714                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3715
3716         return ret;
3717 }
3718
3719 static int
3720 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3721                           uint64_t exec_offset)
3722 {
3723         uint32_t exec_start, exec_len;
3724
3725         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3726         exec_len = (uint32_t) exec->batch_len;
3727
3728         if ((exec_start | exec_len) & 0x7)
3729                 return -EINVAL;
3730
3731         if (!exec_start)
3732                 return -EINVAL;
3733
3734         return 0;
3735 }
3736
3737 static int
3738 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3739                    int count)
3740 {
3741         int i;
3742
3743         for (i = 0; i < count; i++) {
3744                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3745                 int length; /* limited by fault_in_pages_readable() */
3746
3747                 /* First check for malicious input causing overflow */
3748                 if (exec[i].relocation_count >
3749                     INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
3750                         return -EINVAL;
3751
3752                 length = exec[i].relocation_count *
3753                         sizeof(struct drm_i915_gem_relocation_entry);
3754                 if (!access_ok(VERIFY_READ, ptr, length))
3755                         return -EFAULT;
3756
3757                 /* we may also need to update the presumed offsets */
3758                 if (!access_ok(VERIFY_WRITE, ptr, length))
3759                         return -EFAULT;
3760
3761                 if (fault_in_pages_readable(ptr, length))
3762                         return -EFAULT;
3763         }
3764
3765         return 0;
3766 }
3767
3768 static int
3769 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3770                        struct drm_file *file,
3771                        struct drm_i915_gem_execbuffer2 *args,
3772                        struct drm_i915_gem_exec_object2 *exec_list)
3773 {
3774         drm_i915_private_t *dev_priv = dev->dev_private;
3775         struct drm_gem_object **object_list = NULL;
3776         struct drm_gem_object *batch_obj;
3777         struct drm_i915_gem_object *obj_priv;
3778         struct drm_clip_rect *cliprects = NULL;
3779         struct drm_i915_gem_request *request = NULL;
3780         int ret, i, flips;
3781         uint64_t exec_offset;
3782
3783         struct intel_ring_buffer *ring = NULL;
3784
3785         ret = i915_gem_check_is_wedged(dev);
3786         if (ret)
3787                 return ret;
3788
3789         ret = validate_exec_list(exec_list, args->buffer_count);
3790         if (ret)
3791                 return ret;
3792
3793 #if WATCH_EXEC
3794         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3795                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3796 #endif
3797         switch (args->flags & I915_EXEC_RING_MASK) {
3798         case I915_EXEC_DEFAULT:
3799         case I915_EXEC_RENDER:
3800                 ring = &dev_priv->render_ring;
3801                 break;
3802         case I915_EXEC_BSD:
3803                 if (!HAS_BSD(dev)) {
3804                         DRM_ERROR("execbuf with invalid ring (BSD)\n");
3805                         return -EINVAL;
3806                 }
3807                 ring = &dev_priv->bsd_ring;
3808                 break;
3809         case I915_EXEC_BLT:
3810                 if (!HAS_BLT(dev)) {
3811                         DRM_ERROR("execbuf with invalid ring (BLT)\n");
3812                         return -EINVAL;
3813                 }
3814                 ring = &dev_priv->blt_ring;
3815                 break;
3816         default:
3817                 DRM_ERROR("execbuf with unknown ring: %d\n",
3818                           (int)(args->flags & I915_EXEC_RING_MASK));
3819                 return -EINVAL;
3820         }
3821
3822         if (args->buffer_count < 1) {
3823                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3824                 return -EINVAL;
3825         }
3826         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3827         if (object_list == NULL) {
3828                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3829                           args->buffer_count);
3830                 ret = -ENOMEM;
3831                 goto pre_mutex_err;
3832         }
3833
3834         if (args->num_cliprects != 0) {
3835                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3836                                     GFP_KERNEL);
3837                 if (cliprects == NULL) {
3838                         ret = -ENOMEM;
3839                         goto pre_mutex_err;
3840                 }
3841
3842                 ret = copy_from_user(cliprects,
3843                                      (struct drm_clip_rect __user *)
3844                                      (uintptr_t) args->cliprects_ptr,
3845                                      sizeof(*cliprects) * args->num_cliprects);
3846                 if (ret != 0) {
3847                         DRM_ERROR("copy %d cliprects failed: %d\n",
3848                                   args->num_cliprects, ret);
3849                         ret = -EFAULT;
3850                         goto pre_mutex_err;
3851                 }
3852         }
3853
3854         request = kzalloc(sizeof(*request), GFP_KERNEL);
3855         if (request == NULL) {
3856                 ret = -ENOMEM;
3857                 goto pre_mutex_err;
3858         }
3859
3860         ret = i915_mutex_lock_interruptible(dev);
3861         if (ret)
3862                 goto pre_mutex_err;
3863
3864         if (dev_priv->mm.suspended) {
3865                 mutex_unlock(&dev->struct_mutex);
3866                 ret = -EBUSY;
3867                 goto pre_mutex_err;
3868         }
3869
3870         /* Look up object handles */
3871         for (i = 0; i < args->buffer_count; i++) {
3872                 object_list[i] = drm_gem_object_lookup(dev, file,
3873                                                        exec_list[i].handle);
3874                 if (object_list[i] == NULL) {
3875                         DRM_ERROR("Invalid object handle %d at index %d\n",
3876                                    exec_list[i].handle, i);
3877                         /* prevent error path from reading uninitialized data */
3878                         args->buffer_count = i + 1;
3879                         ret = -ENOENT;
3880                         goto err;
3881                 }
3882
3883                 obj_priv = to_intel_bo(object_list[i]);
3884                 if (obj_priv->in_execbuffer) {
3885                         DRM_ERROR("Object %p appears more than once in object list\n",
3886                                    object_list[i]);
3887                         /* prevent error path from reading uninitialized data */
3888                         args->buffer_count = i + 1;
3889                         ret = -EINVAL;
3890                         goto err;
3891                 }
3892                 obj_priv->in_execbuffer = true;
3893         }
3894
3895         /* Move the objects en-masse into the GTT, evicting if necessary. */
3896         ret = i915_gem_execbuffer_reserve(dev, file,
3897                                           object_list, exec_list,
3898                                           args->buffer_count);
3899         if (ret)
3900                 goto err;
3901
3902         /* The objects are in their final locations, apply the relocations. */
3903         ret = i915_gem_execbuffer_relocate(dev, file,
3904                                            object_list, exec_list,
3905                                            args->buffer_count);
3906         if (ret) {
3907                 if (ret == -EFAULT) {
3908                         ret = i915_gem_execbuffer_relocate_slow(dev, file,
3909                                                                 object_list,
3910                                                                 exec_list,
3911                                                                 args->buffer_count);
3912                         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
3913                 }
3914                 if (ret)
3915                         goto err;
3916         }
3917
3918         /* Set the pending read domains for the batch buffer to COMMAND */
3919         batch_obj = object_list[args->buffer_count-1];
3920         if (batch_obj->pending_write_domain) {
3921                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3922                 ret = -EINVAL;
3923                 goto err;
3924         }
3925         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3926
3927         /* Sanity check the batch buffer */
3928         exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3929         ret = i915_gem_check_execbuffer(args, exec_offset);
3930         if (ret != 0) {
3931                 DRM_ERROR("execbuf with invalid offset/length\n");
3932                 goto err;
3933         }
3934
3935         ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
3936                                               object_list, args->buffer_count);
3937         if (ret)
3938                 goto err;
3939
3940         for (i = 0; i < args->buffer_count; i++) {
3941                 struct drm_gem_object *obj = object_list[i];
3942                 uint32_t old_write_domain = obj->write_domain;
3943                 obj->write_domain = obj->pending_write_domain;
3944                 trace_i915_gem_object_change_domain(obj,
3945                                                     obj->read_domains,
3946                                                     old_write_domain);
3947         }
3948
3949 #if WATCH_COHERENCY
3950         for (i = 0; i < args->buffer_count; i++) {
3951                 i915_gem_object_check_coherency(object_list[i],
3952                                                 exec_list[i].handle);
3953         }
3954 #endif
3955
3956 #if WATCH_EXEC
3957         i915_gem_dump_object(batch_obj,
3958                               args->batch_len,
3959                               __func__,
3960                               ~0);
3961 #endif
3962
3963         /* Check for any pending flips. As we only maintain a flip queue depth
3964          * of 1, we can simply insert a WAIT for the next display flip prior
3965          * to executing the batch and avoid stalling the CPU.
3966          */
3967         flips = 0;
3968         for (i = 0; i < args->buffer_count; i++) {
3969                 if (object_list[i]->write_domain)
3970                         flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3971         }
3972         if (flips) {
3973                 int plane, flip_mask;
3974
3975                 for (plane = 0; flips >> plane; plane++) {
3976                         if (((flips >> plane) & 1) == 0)
3977                                 continue;
3978
3979                         if (plane)
3980                                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3981                         else
3982                                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3983
3984                         intel_ring_begin(dev, ring, 2);
3985                         intel_ring_emit(dev, ring,
3986                                         MI_WAIT_FOR_EVENT | flip_mask);
3987                         intel_ring_emit(dev, ring, MI_NOOP);
3988                         intel_ring_advance(dev, ring);
3989                 }
3990         }
3991
3992         /* Exec the batchbuffer */
3993         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3994                                             cliprects, exec_offset);
3995         if (ret) {
3996                 DRM_ERROR("dispatch failed %d\n", ret);
3997                 goto err;
3998         }
3999
4000         /*
4001          * Ensure that the commands in the batch buffer are
4002          * finished before the interrupt fires
4003          */
4004         i915_retire_commands(dev, ring);
4005
4006         for (i = 0; i < args->buffer_count; i++) {
4007                 struct drm_gem_object *obj = object_list[i];
4008
4009                 i915_gem_object_move_to_active(obj, ring);
4010                 if (obj->write_domain)
4011                         list_move_tail(&to_intel_bo(obj)->gpu_write_list,
4012                                        &ring->gpu_write_list);
4013         }
4014
4015         i915_add_request(dev, file, request, ring);
4016         request = NULL;
4017
4018 err:
4019         for (i = 0; i < args->buffer_count; i++) {
4020                 if (object_list[i]) {
4021                         obj_priv = to_intel_bo(object_list[i]);
4022                         obj_priv->in_execbuffer = false;
4023                 }
4024                 drm_gem_object_unreference(object_list[i]);
4025         }
4026
4027         mutex_unlock(&dev->struct_mutex);
4028
4029 pre_mutex_err:
4030         drm_free_large(object_list);
4031         kfree(cliprects);
4032         kfree(request);
4033
4034         return ret;
4035 }
4036
4037 /*
4038  * Legacy execbuffer just creates an exec2 list from the original exec object
4039  * list array and passes it to the real function.
4040  */
4041 int
4042 i915_gem_execbuffer(struct drm_device *dev, void *data,
4043                     struct drm_file *file_priv)
4044 {
4045         struct drm_i915_gem_execbuffer *args = data;
4046         struct drm_i915_gem_execbuffer2 exec2;
4047         struct drm_i915_gem_exec_object *exec_list = NULL;
4048         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4049         int ret, i;
4050
4051 #if WATCH_EXEC
4052         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4053                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4054 #endif
4055
4056         if (args->buffer_count < 1) {
4057                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4058                 return -EINVAL;
4059         }
4060
4061         /* Copy in the exec list from userland */
4062         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4063         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4064         if (exec_list == NULL || exec2_list == NULL) {
4065                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4066                           args->buffer_count);
4067                 drm_free_large(exec_list);
4068                 drm_free_large(exec2_list);
4069                 return -ENOMEM;
4070         }
4071         ret = copy_from_user(exec_list,
4072                              (struct drm_i915_relocation_entry __user *)
4073                              (uintptr_t) args->buffers_ptr,
4074                              sizeof(*exec_list) * args->buffer_count);
4075         if (ret != 0) {
4076                 DRM_ERROR("copy %d exec entries failed %d\n",
4077                           args->buffer_count, ret);
4078                 drm_free_large(exec_list);
4079                 drm_free_large(exec2_list);
4080                 return -EFAULT;
4081         }
4082
4083         for (i = 0; i < args->buffer_count; i++) {
4084                 exec2_list[i].handle = exec_list[i].handle;
4085                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4086                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4087                 exec2_list[i].alignment = exec_list[i].alignment;
4088                 exec2_list[i].offset = exec_list[i].offset;
4089                 if (INTEL_INFO(dev)->gen < 4)
4090                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4091                 else
4092                         exec2_list[i].flags = 0;
4093         }
4094
4095         exec2.buffers_ptr = args->buffers_ptr;
4096         exec2.buffer_count = args->buffer_count;
4097         exec2.batch_start_offset = args->batch_start_offset;
4098         exec2.batch_len = args->batch_len;
4099         exec2.DR1 = args->DR1;
4100         exec2.DR4 = args->DR4;
4101         exec2.num_cliprects = args->num_cliprects;
4102         exec2.cliprects_ptr = args->cliprects_ptr;
4103         exec2.flags = I915_EXEC_RENDER;
4104
4105         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4106         if (!ret) {
4107                 /* Copy the new buffer offsets back to the user's exec list. */
4108                 for (i = 0; i < args->buffer_count; i++)
4109                         exec_list[i].offset = exec2_list[i].offset;
4110                 /* ... and back out to userspace */
4111                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4112                                    (uintptr_t) args->buffers_ptr,
4113                                    exec_list,
4114                                    sizeof(*exec_list) * args->buffer_count);
4115                 if (ret) {
4116                         ret = -EFAULT;
4117                         DRM_ERROR("failed to copy %d exec entries "
4118                                   "back to user (%d)\n",
4119                                   args->buffer_count, ret);
4120                 }
4121         }
4122
4123         drm_free_large(exec_list);
4124         drm_free_large(exec2_list);
4125         return ret;
4126 }
4127
4128 int
4129 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4130                      struct drm_file *file_priv)
4131 {
4132         struct drm_i915_gem_execbuffer2 *args = data;
4133         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4134         int ret;
4135
4136 #if WATCH_EXEC
4137         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4138                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4139 #endif
4140
4141         if (args->buffer_count < 1) {
4142                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4143                 return -EINVAL;
4144         }
4145
4146         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4147         if (exec2_list == NULL) {
4148                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4149                           args->buffer_count);
4150                 return -ENOMEM;
4151         }
4152         ret = copy_from_user(exec2_list,
4153                              (struct drm_i915_relocation_entry __user *)
4154                              (uintptr_t) args->buffers_ptr,
4155                              sizeof(*exec2_list) * args->buffer_count);
4156         if (ret != 0) {
4157                 DRM_ERROR("copy %d exec entries failed %d\n",
4158                           args->buffer_count, ret);
4159                 drm_free_large(exec2_list);
4160                 return -EFAULT;
4161         }
4162
4163         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4164         if (!ret) {
4165                 /* Copy the new buffer offsets back to the user's exec list. */
4166                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4167                                    (uintptr_t) args->buffers_ptr,
4168                                    exec2_list,
4169                                    sizeof(*exec2_list) * args->buffer_count);
4170                 if (ret) {
4171                         ret = -EFAULT;
4172                         DRM_ERROR("failed to copy %d exec entries "
4173                                   "back to user (%d)\n",
4174                                   args->buffer_count, ret);
4175                 }
4176         }
4177
4178         drm_free_large(exec2_list);
4179         return ret;
4180 }
4181
4182 int
4183 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4184 {
4185         struct drm_device *dev = obj->dev;
4186         struct drm_i915_private *dev_priv = dev->dev_private;
4187         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4188         int ret;
4189
4190         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4191         WARN_ON(i915_verify_lists(dev));
4192
4193         if (obj_priv->gtt_space != NULL) {
4194                 if (alignment == 0)
4195                         alignment = i915_gem_get_gtt_alignment(obj);
4196                 if (obj_priv->gtt_offset & (alignment - 1)) {
4197                         WARN(obj_priv->pin_count,
4198                              "bo is already pinned with incorrect alignment: offset=%x, req.alignment=%x\n",
4199                              obj_priv->gtt_offset, alignment);
4200                         ret = i915_gem_object_unbind(obj);
4201                         if (ret)
4202                                 return ret;
4203                 }
4204         }
4205
4206         if (obj_priv->gtt_space == NULL) {
4207                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4208                 if (ret)
4209                         return ret;
4210         }
4211
4212         obj_priv->pin_count++;
4213
4214         /* If the object is not active and not pending a flush,
4215          * remove it from the inactive list
4216          */
4217         if (obj_priv->pin_count == 1) {
4218                 i915_gem_info_add_pin(dev_priv, obj->size);
4219                 if (!obj_priv->active)
4220                         list_move_tail(&obj_priv->mm_list,
4221                                        &dev_priv->mm.pinned_list);
4222         }
4223
4224         WARN_ON(i915_verify_lists(dev));
4225         return 0;
4226 }
4227
4228 void
4229 i915_gem_object_unpin(struct drm_gem_object *obj)
4230 {
4231         struct drm_device *dev = obj->dev;
4232         drm_i915_private_t *dev_priv = dev->dev_private;
4233         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4234
4235         WARN_ON(i915_verify_lists(dev));
4236         obj_priv->pin_count--;
4237         BUG_ON(obj_priv->pin_count < 0);
4238         BUG_ON(obj_priv->gtt_space == NULL);
4239
4240         /* If the object is no longer pinned, and is
4241          * neither active nor being flushed, then stick it on
4242          * the inactive list
4243          */
4244         if (obj_priv->pin_count == 0) {
4245                 if (!obj_priv->active)
4246                         list_move_tail(&obj_priv->mm_list,
4247                                        &dev_priv->mm.inactive_list);
4248                 i915_gem_info_remove_pin(dev_priv, obj->size);
4249         }
4250         WARN_ON(i915_verify_lists(dev));
4251 }
4252
4253 int
4254 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4255                    struct drm_file *file_priv)
4256 {
4257         struct drm_i915_gem_pin *args = data;
4258         struct drm_gem_object *obj;
4259         struct drm_i915_gem_object *obj_priv;
4260         int ret;
4261
4262         ret = i915_mutex_lock_interruptible(dev);
4263         if (ret)
4264                 return ret;
4265
4266         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4267         if (obj == NULL) {
4268                 ret = -ENOENT;
4269                 goto unlock;
4270         }
4271         obj_priv = to_intel_bo(obj);
4272
4273         if (obj_priv->madv != I915_MADV_WILLNEED) {
4274                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4275                 ret = -EINVAL;
4276                 goto out;
4277         }
4278
4279         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4280                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4281                           args->handle);
4282                 ret = -EINVAL;
4283                 goto out;
4284         }
4285
4286         obj_priv->user_pin_count++;
4287         obj_priv->pin_filp = file_priv;
4288         if (obj_priv->user_pin_count == 1) {
4289                 ret = i915_gem_object_pin(obj, args->alignment);
4290                 if (ret)
4291                         goto out;
4292         }
4293
4294         /* XXX - flush the CPU caches for pinned objects
4295          * as the X server doesn't manage domains yet
4296          */
4297         i915_gem_object_flush_cpu_write_domain(obj);
4298         args->offset = obj_priv->gtt_offset;
4299 out:
4300         drm_gem_object_unreference(obj);
4301 unlock:
4302         mutex_unlock(&dev->struct_mutex);
4303         return ret;
4304 }
4305
4306 int
4307 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4308                      struct drm_file *file_priv)
4309 {
4310         struct drm_i915_gem_pin *args = data;
4311         struct drm_gem_object *obj;
4312         struct drm_i915_gem_object *obj_priv;
4313         int ret;
4314
4315         ret = i915_mutex_lock_interruptible(dev);
4316         if (ret)
4317                 return ret;
4318
4319         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4320         if (obj == NULL) {
4321                 ret = -ENOENT;
4322                 goto unlock;
4323         }
4324         obj_priv = to_intel_bo(obj);
4325
4326         if (obj_priv->pin_filp != file_priv) {
4327                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4328                           args->handle);
4329                 ret = -EINVAL;
4330                 goto out;
4331         }
4332         obj_priv->user_pin_count--;
4333         if (obj_priv->user_pin_count == 0) {
4334                 obj_priv->pin_filp = NULL;
4335                 i915_gem_object_unpin(obj);
4336         }
4337
4338 out:
4339         drm_gem_object_unreference(obj);
4340 unlock:
4341         mutex_unlock(&dev->struct_mutex);
4342         return ret;
4343 }
4344
4345 int
4346 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4347                     struct drm_file *file_priv)
4348 {
4349         struct drm_i915_gem_busy *args = data;
4350         struct drm_gem_object *obj;
4351         struct drm_i915_gem_object *obj_priv;
4352         int ret;
4353
4354         ret = i915_mutex_lock_interruptible(dev);
4355         if (ret)
4356                 return ret;
4357
4358         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4359         if (obj == NULL) {
4360                 ret = -ENOENT;
4361                 goto unlock;
4362         }
4363         obj_priv = to_intel_bo(obj);
4364
4365         /* Count all active objects as busy, even if they are currently not used
4366          * by the gpu. Users of this interface expect objects to eventually
4367          * become non-busy without any further actions, therefore emit any
4368          * necessary flushes here.
4369          */
4370         args->busy = obj_priv->active;
4371         if (args->busy) {
4372                 /* Unconditionally flush objects, even when the gpu still uses this
4373                  * object. Userspace calling this function indicates that it wants to
4374                  * use this buffer rather sooner than later, so issuing the required
4375                  * flush earlier is beneficial.
4376                  */
4377                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4378                         i915_gem_flush_ring(dev, file_priv,
4379                                             obj_priv->ring,
4380                                             0, obj->write_domain);
4381
4382                 /* Update the active list for the hardware's current position.
4383                  * Otherwise this only updates on a delayed timer or when irqs
4384                  * are actually unmasked, and our working set ends up being
4385                  * larger than required.
4386                  */
4387                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4388
4389                 args->busy = obj_priv->active;
4390         }
4391
4392         drm_gem_object_unreference(obj);
4393 unlock:
4394         mutex_unlock(&dev->struct_mutex);
4395         return ret;
4396 }
4397
4398 int
4399 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4400                         struct drm_file *file_priv)
4401 {
4402     return i915_gem_ring_throttle(dev, file_priv);
4403 }
4404
4405 int
4406 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4407                        struct drm_file *file_priv)
4408 {
4409         struct drm_i915_gem_madvise *args = data;
4410         struct drm_gem_object *obj;
4411         struct drm_i915_gem_object *obj_priv;
4412         int ret;
4413
4414         switch (args->madv) {
4415         case I915_MADV_DONTNEED:
4416         case I915_MADV_WILLNEED:
4417             break;
4418         default:
4419             return -EINVAL;
4420         }
4421
4422         ret = i915_mutex_lock_interruptible(dev);
4423         if (ret)
4424                 return ret;
4425
4426         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4427         if (obj == NULL) {
4428                 ret = -ENOENT;
4429                 goto unlock;
4430         }
4431         obj_priv = to_intel_bo(obj);
4432
4433         if (obj_priv->pin_count) {
4434                 ret = -EINVAL;
4435                 goto out;
4436         }
4437
4438         if (obj_priv->madv != __I915_MADV_PURGED)
4439                 obj_priv->madv = args->madv;
4440
4441         /* if the object is no longer bound, discard its backing storage */
4442         if (i915_gem_object_is_purgeable(obj_priv) &&
4443             obj_priv->gtt_space == NULL)
4444                 i915_gem_object_truncate(obj);
4445
4446         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4447
4448 out:
4449         drm_gem_object_unreference(obj);
4450 unlock:
4451         mutex_unlock(&dev->struct_mutex);
4452         return ret;
4453 }
4454
4455 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4456                                               size_t size)
4457 {
4458         struct drm_i915_private *dev_priv = dev->dev_private;
4459         struct drm_i915_gem_object *obj;
4460
4461         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4462         if (obj == NULL)
4463                 return NULL;
4464
4465         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4466                 kfree(obj);
4467                 return NULL;
4468         }
4469
4470         i915_gem_info_add_obj(dev_priv, size);
4471
4472         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4473         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4474
4475         obj->agp_type = AGP_USER_MEMORY;
4476         obj->base.driver_private = NULL;
4477         obj->fence_reg = I915_FENCE_REG_NONE;
4478         INIT_LIST_HEAD(&obj->mm_list);
4479         INIT_LIST_HEAD(&obj->ring_list);
4480         INIT_LIST_HEAD(&obj->gpu_write_list);
4481         obj->madv = I915_MADV_WILLNEED;
4482
4483         return &obj->base;
4484 }
4485
4486 int i915_gem_init_object(struct drm_gem_object *obj)
4487 {
4488         BUG();
4489
4490         return 0;
4491 }
4492
4493 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4494 {
4495         struct drm_device *dev = obj->dev;
4496         drm_i915_private_t *dev_priv = dev->dev_private;
4497         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4498         int ret;
4499
4500         ret = i915_gem_object_unbind(obj);
4501         if (ret == -ERESTARTSYS) {
4502                 list_move(&obj_priv->mm_list,
4503                           &dev_priv->mm.deferred_free_list);
4504                 return;
4505         }
4506
4507         if (obj_priv->mmap_offset)
4508                 i915_gem_free_mmap_offset(obj);
4509
4510         drm_gem_object_release(obj);
4511         i915_gem_info_remove_obj(dev_priv, obj->size);
4512
4513         kfree(obj_priv->page_cpu_valid);
4514         kfree(obj_priv->bit_17);
4515         kfree(obj_priv);
4516 }
4517
4518 void i915_gem_free_object(struct drm_gem_object *obj)
4519 {
4520         struct drm_device *dev = obj->dev;
4521         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4522
4523         trace_i915_gem_object_destroy(obj);
4524
4525         while (obj_priv->pin_count > 0)
4526                 i915_gem_object_unpin(obj);
4527
4528         if (obj_priv->phys_obj)
4529                 i915_gem_detach_phys_object(dev, obj);
4530
4531         i915_gem_free_object_tail(obj);
4532 }
4533
4534 int
4535 i915_gem_idle(struct drm_device *dev)
4536 {
4537         drm_i915_private_t *dev_priv = dev->dev_private;
4538         int ret;
4539
4540         mutex_lock(&dev->struct_mutex);
4541
4542         if (dev_priv->mm.suspended) {
4543                 mutex_unlock(&dev->struct_mutex);
4544                 return 0;
4545         }
4546
4547         ret = i915_gpu_idle(dev);
4548         if (ret) {
4549                 mutex_unlock(&dev->struct_mutex);
4550                 return ret;
4551         }
4552
4553         /* Under UMS, be paranoid and evict. */
4554         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4555                 ret = i915_gem_evict_inactive(dev);
4556                 if (ret) {
4557                         mutex_unlock(&dev->struct_mutex);
4558                         return ret;
4559                 }
4560         }
4561
4562         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4563          * We need to replace this with a semaphore, or something.
4564          * And not confound mm.suspended!
4565          */
4566         dev_priv->mm.suspended = 1;
4567         del_timer_sync(&dev_priv->hangcheck_timer);
4568
4569         i915_kernel_lost_context(dev);
4570         i915_gem_cleanup_ringbuffer(dev);
4571
4572         mutex_unlock(&dev->struct_mutex);
4573
4574         /* Cancel the retire work handler, which should be idle now. */
4575         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4576
4577         return 0;
4578 }
4579
4580 /*
4581  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4582  * over cache flushing.
4583  */
4584 static int
4585 i915_gem_init_pipe_control(struct drm_device *dev)
4586 {
4587         drm_i915_private_t *dev_priv = dev->dev_private;
4588         struct drm_gem_object *obj;
4589         struct drm_i915_gem_object *obj_priv;
4590         int ret;
4591
4592         obj = i915_gem_alloc_object(dev, 4096);
4593         if (obj == NULL) {
4594                 DRM_ERROR("Failed to allocate seqno page\n");
4595                 ret = -ENOMEM;
4596                 goto err;
4597         }
4598         obj_priv = to_intel_bo(obj);
4599         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4600
4601         ret = i915_gem_object_pin(obj, 4096);
4602         if (ret)
4603                 goto err_unref;
4604
4605         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4606         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4607         if (dev_priv->seqno_page == NULL)
4608                 goto err_unpin;
4609
4610         dev_priv->seqno_obj = obj;
4611         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4612
4613         return 0;
4614
4615 err_unpin:
4616         i915_gem_object_unpin(obj);
4617 err_unref:
4618         drm_gem_object_unreference(obj);
4619 err:
4620         return ret;
4621 }
4622
4623
4624 static void
4625 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4626 {
4627         drm_i915_private_t *dev_priv = dev->dev_private;
4628         struct drm_gem_object *obj;
4629         struct drm_i915_gem_object *obj_priv;
4630
4631         obj = dev_priv->seqno_obj;
4632         obj_priv = to_intel_bo(obj);
4633         kunmap(obj_priv->pages[0]);
4634         i915_gem_object_unpin(obj);
4635         drm_gem_object_unreference(obj);
4636         dev_priv->seqno_obj = NULL;
4637
4638         dev_priv->seqno_page = NULL;
4639 }
4640
4641 int
4642 i915_gem_init_ringbuffer(struct drm_device *dev)
4643 {
4644         drm_i915_private_t *dev_priv = dev->dev_private;
4645         int ret;
4646
4647         if (HAS_PIPE_CONTROL(dev)) {
4648                 ret = i915_gem_init_pipe_control(dev);
4649                 if (ret)
4650                         return ret;
4651         }
4652
4653         ret = intel_init_render_ring_buffer(dev);
4654         if (ret)
4655                 goto cleanup_pipe_control;
4656
4657         if (HAS_BSD(dev)) {
4658                 ret = intel_init_bsd_ring_buffer(dev);
4659                 if (ret)
4660                         goto cleanup_render_ring;
4661         }
4662
4663         if (HAS_BLT(dev)) {
4664                 ret = intel_init_blt_ring_buffer(dev);
4665                 if (ret)
4666                         goto cleanup_bsd_ring;
4667         }
4668
4669         dev_priv->next_seqno = 1;
4670
4671         return 0;
4672
4673 cleanup_bsd_ring:
4674         intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4675 cleanup_render_ring:
4676         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4677 cleanup_pipe_control:
4678         if (HAS_PIPE_CONTROL(dev))
4679                 i915_gem_cleanup_pipe_control(dev);
4680         return ret;
4681 }
4682
4683 void
4684 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4685 {
4686         drm_i915_private_t *dev_priv = dev->dev_private;
4687
4688         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4689         intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4690         intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring);
4691         if (HAS_PIPE_CONTROL(dev))
4692                 i915_gem_cleanup_pipe_control(dev);
4693 }
4694
4695 int
4696 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4697                        struct drm_file *file_priv)
4698 {
4699         drm_i915_private_t *dev_priv = dev->dev_private;
4700         int ret;
4701
4702         if (drm_core_check_feature(dev, DRIVER_MODESET))
4703                 return 0;
4704
4705         if (atomic_read(&dev_priv->mm.wedged)) {
4706                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4707                 atomic_set(&dev_priv->mm.wedged, 0);
4708         }
4709
4710         mutex_lock(&dev->struct_mutex);
4711         dev_priv->mm.suspended = 0;
4712
4713         ret = i915_gem_init_ringbuffer(dev);
4714         if (ret != 0) {
4715                 mutex_unlock(&dev->struct_mutex);
4716                 return ret;
4717         }
4718
4719         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4720         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4721         BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4722         BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4723         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4724         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4725         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4726         BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4727         BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4728         mutex_unlock(&dev->struct_mutex);
4729
4730         ret = drm_irq_install(dev);
4731         if (ret)
4732                 goto cleanup_ringbuffer;
4733
4734         return 0;
4735
4736 cleanup_ringbuffer:
4737         mutex_lock(&dev->struct_mutex);
4738         i915_gem_cleanup_ringbuffer(dev);
4739         dev_priv->mm.suspended = 1;
4740         mutex_unlock(&dev->struct_mutex);
4741
4742         return ret;
4743 }
4744
4745 int
4746 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4747                        struct drm_file *file_priv)
4748 {
4749         if (drm_core_check_feature(dev, DRIVER_MODESET))
4750                 return 0;
4751
4752         drm_irq_uninstall(dev);
4753         return i915_gem_idle(dev);
4754 }
4755
4756 void
4757 i915_gem_lastclose(struct drm_device *dev)
4758 {
4759         int ret;
4760
4761         if (drm_core_check_feature(dev, DRIVER_MODESET))
4762                 return;
4763
4764         ret = i915_gem_idle(dev);
4765         if (ret)
4766                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4767 }
4768
4769 static void
4770 init_ring_lists(struct intel_ring_buffer *ring)
4771 {
4772         INIT_LIST_HEAD(&ring->active_list);
4773         INIT_LIST_HEAD(&ring->request_list);
4774         INIT_LIST_HEAD(&ring->gpu_write_list);
4775 }
4776
4777 void
4778 i915_gem_load(struct drm_device *dev)
4779 {
4780         int i;
4781         drm_i915_private_t *dev_priv = dev->dev_private;
4782
4783         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4784         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4785         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4786         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4787         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4788         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4789         init_ring_lists(&dev_priv->render_ring);
4790         init_ring_lists(&dev_priv->bsd_ring);
4791         init_ring_lists(&dev_priv->blt_ring);
4792         for (i = 0; i < 16; i++)
4793                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4794         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4795                           i915_gem_retire_work_handler);
4796         init_completion(&dev_priv->error_completion);
4797         spin_lock(&shrink_list_lock);
4798         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4799         spin_unlock(&shrink_list_lock);
4800
4801         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4802         if (IS_GEN3(dev)) {
4803                 u32 tmp = I915_READ(MI_ARB_STATE);
4804                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4805                         /* arb state is a masked write, so set bit + bit in mask */
4806                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4807                         I915_WRITE(MI_ARB_STATE, tmp);
4808                 }
4809         }
4810
4811         /* Old X drivers will take 0-2 for front, back, depth buffers */
4812         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4813                 dev_priv->fence_reg_start = 3;
4814
4815         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4816                 dev_priv->num_fence_regs = 16;
4817         else
4818                 dev_priv->num_fence_regs = 8;
4819
4820         /* Initialize fence registers to zero */
4821         switch (INTEL_INFO(dev)->gen) {
4822         case 6:
4823                 for (i = 0; i < 16; i++)
4824                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4825                 break;
4826         case 5:
4827         case 4:
4828                 for (i = 0; i < 16; i++)
4829                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4830                 break;
4831         case 3:
4832                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4833                         for (i = 0; i < 8; i++)
4834                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4835         case 2:
4836                 for (i = 0; i < 8; i++)
4837                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4838                 break;
4839         }
4840         i915_gem_detect_bit_6_swizzle(dev);
4841         init_waitqueue_head(&dev_priv->pending_flip_queue);
4842 }
4843
4844 /*
4845  * Create a physically contiguous memory object for this object
4846  * e.g. for cursor + overlay regs
4847  */
4848 static int i915_gem_init_phys_object(struct drm_device *dev,
4849                                      int id, int size, int align)
4850 {
4851         drm_i915_private_t *dev_priv = dev->dev_private;
4852         struct drm_i915_gem_phys_object *phys_obj;
4853         int ret;
4854
4855         if (dev_priv->mm.phys_objs[id - 1] || !size)
4856                 return 0;
4857
4858         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4859         if (!phys_obj)
4860                 return -ENOMEM;
4861
4862         phys_obj->id = id;
4863
4864         phys_obj->handle = drm_pci_alloc(dev, size, align);
4865         if (!phys_obj->handle) {
4866                 ret = -ENOMEM;
4867                 goto kfree_obj;
4868         }
4869 #ifdef CONFIG_X86
4870         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4871 #endif
4872
4873         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4874
4875         return 0;
4876 kfree_obj:
4877         kfree(phys_obj);
4878         return ret;
4879 }
4880
4881 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4882 {
4883         drm_i915_private_t *dev_priv = dev->dev_private;
4884         struct drm_i915_gem_phys_object *phys_obj;
4885
4886         if (!dev_priv->mm.phys_objs[id - 1])
4887                 return;
4888
4889         phys_obj = dev_priv->mm.phys_objs[id - 1];
4890         if (phys_obj->cur_obj) {
4891                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4892         }
4893
4894 #ifdef CONFIG_X86
4895         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4896 #endif
4897         drm_pci_free(dev, phys_obj->handle);
4898         kfree(phys_obj);
4899         dev_priv->mm.phys_objs[id - 1] = NULL;
4900 }
4901
4902 void i915_gem_free_all_phys_object(struct drm_device *dev)
4903 {
4904         int i;
4905
4906         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4907                 i915_gem_free_phys_object(dev, i);
4908 }
4909
4910 void i915_gem_detach_phys_object(struct drm_device *dev,
4911                                  struct drm_gem_object *obj)
4912 {
4913         struct drm_i915_gem_object *obj_priv;
4914         int i;
4915         int ret;
4916         int page_count;
4917
4918         obj_priv = to_intel_bo(obj);
4919         if (!obj_priv->phys_obj)
4920                 return;
4921
4922         ret = i915_gem_object_get_pages(obj, 0);
4923         if (ret)
4924                 goto out;
4925
4926         page_count = obj->size / PAGE_SIZE;
4927
4928         for (i = 0; i < page_count; i++) {
4929                 char *dst = kmap_atomic(obj_priv->pages[i]);
4930                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4931
4932                 memcpy(dst, src, PAGE_SIZE);
4933                 kunmap_atomic(dst);
4934         }
4935         drm_clflush_pages(obj_priv->pages, page_count);
4936         drm_agp_chipset_flush(dev);
4937
4938         i915_gem_object_put_pages(obj);
4939 out:
4940         obj_priv->phys_obj->cur_obj = NULL;
4941         obj_priv->phys_obj = NULL;
4942 }
4943
4944 int
4945 i915_gem_attach_phys_object(struct drm_device *dev,
4946                             struct drm_gem_object *obj,
4947                             int id,
4948                             int align)
4949 {
4950         drm_i915_private_t *dev_priv = dev->dev_private;
4951         struct drm_i915_gem_object *obj_priv;
4952         int ret = 0;
4953         int page_count;
4954         int i;
4955
4956         if (id > I915_MAX_PHYS_OBJECT)
4957                 return -EINVAL;
4958
4959         obj_priv = to_intel_bo(obj);
4960
4961         if (obj_priv->phys_obj) {
4962                 if (obj_priv->phys_obj->id == id)
4963                         return 0;
4964                 i915_gem_detach_phys_object(dev, obj);
4965         }
4966
4967         /* create a new object */
4968         if (!dev_priv->mm.phys_objs[id - 1]) {
4969                 ret = i915_gem_init_phys_object(dev, id,
4970                                                 obj->size, align);
4971                 if (ret) {
4972                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4973                         goto out;
4974                 }
4975         }
4976
4977         /* bind to the object */
4978         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4979         obj_priv->phys_obj->cur_obj = obj;
4980
4981         ret = i915_gem_object_get_pages(obj, 0);
4982         if (ret) {
4983                 DRM_ERROR("failed to get page list\n");
4984                 goto out;
4985         }
4986
4987         page_count = obj->size / PAGE_SIZE;
4988
4989         for (i = 0; i < page_count; i++) {
4990                 char *src = kmap_atomic(obj_priv->pages[i]);
4991                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4992
4993                 memcpy(dst, src, PAGE_SIZE);
4994                 kunmap_atomic(src);
4995         }
4996
4997         i915_gem_object_put_pages(obj);
4998
4999         return 0;
5000 out:
5001         return ret;
5002 }
5003
5004 static int
5005 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
5006                      struct drm_i915_gem_pwrite *args,
5007                      struct drm_file *file_priv)
5008 {
5009         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
5010         void *vaddr = obj_priv->phys_obj->handle->vaddr + args->offset;
5011         char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
5012
5013         DRM_DEBUG_DRIVER("vaddr %p, %lld\n", vaddr, args->size);
5014
5015         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
5016                 unsigned long unwritten;
5017
5018                 /* The physical object once assigned is fixed for the lifetime
5019                  * of the obj, so we can safely drop the lock and continue
5020                  * to access vaddr.
5021                  */
5022                 mutex_unlock(&dev->struct_mutex);
5023                 unwritten = copy_from_user(vaddr, user_data, args->size);
5024                 mutex_lock(&dev->struct_mutex);
5025                 if (unwritten)
5026                         return -EFAULT;
5027         }
5028
5029         drm_agp_chipset_flush(dev);
5030         return 0;
5031 }
5032
5033 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5034 {
5035         struct drm_i915_file_private *file_priv = file->driver_priv;
5036
5037         /* Clean up our request list when the client is going away, so that
5038          * later retire_requests won't dereference our soon-to-be-gone
5039          * file_priv.
5040          */
5041         spin_lock(&file_priv->mm.lock);
5042         while (!list_empty(&file_priv->mm.request_list)) {
5043                 struct drm_i915_gem_request *request;
5044
5045                 request = list_first_entry(&file_priv->mm.request_list,
5046                                            struct drm_i915_gem_request,
5047                                            client_list);
5048                 list_del(&request->client_list);
5049                 request->file_priv = NULL;
5050         }
5051         spin_unlock(&file_priv->mm.lock);
5052 }
5053
5054 static int
5055 i915_gpu_is_active(struct drm_device *dev)
5056 {
5057         drm_i915_private_t *dev_priv = dev->dev_private;
5058         int lists_empty;
5059
5060         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5061                       list_empty(&dev_priv->mm.active_list);
5062
5063         return !lists_empty;
5064 }
5065
5066 static int
5067 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
5068 {
5069         drm_i915_private_t *dev_priv, *next_dev;
5070         struct drm_i915_gem_object *obj_priv, *next_obj;
5071         int cnt = 0;
5072         int would_deadlock = 1;
5073
5074         /* "fast-path" to count number of available objects */
5075         if (nr_to_scan == 0) {
5076                 spin_lock(&shrink_list_lock);
5077                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5078                         struct drm_device *dev = dev_priv->dev;
5079
5080                         if (mutex_trylock(&dev->struct_mutex)) {
5081                                 list_for_each_entry(obj_priv,
5082                                                     &dev_priv->mm.inactive_list,
5083                                                     mm_list)
5084                                         cnt++;
5085                                 mutex_unlock(&dev->struct_mutex);
5086                         }
5087                 }
5088                 spin_unlock(&shrink_list_lock);
5089
5090                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5091         }
5092
5093         spin_lock(&shrink_list_lock);
5094
5095 rescan:
5096         /* first scan for clean buffers */
5097         list_for_each_entry_safe(dev_priv, next_dev,
5098                                  &shrink_list, mm.shrink_list) {
5099                 struct drm_device *dev = dev_priv->dev;
5100
5101                 if (! mutex_trylock(&dev->struct_mutex))
5102                         continue;
5103
5104                 spin_unlock(&shrink_list_lock);
5105                 i915_gem_retire_requests(dev);
5106
5107                 list_for_each_entry_safe(obj_priv, next_obj,
5108                                          &dev_priv->mm.inactive_list,
5109                                          mm_list) {
5110                         if (i915_gem_object_is_purgeable(obj_priv)) {
5111                                 i915_gem_object_unbind(&obj_priv->base);
5112                                 if (--nr_to_scan <= 0)
5113                                         break;
5114                         }
5115                 }
5116
5117                 spin_lock(&shrink_list_lock);
5118                 mutex_unlock(&dev->struct_mutex);
5119
5120                 would_deadlock = 0;
5121
5122                 if (nr_to_scan <= 0)
5123                         break;
5124         }
5125
5126         /* second pass, evict/count anything still on the inactive list */
5127         list_for_each_entry_safe(dev_priv, next_dev,
5128                                  &shrink_list, mm.shrink_list) {
5129                 struct drm_device *dev = dev_priv->dev;
5130
5131                 if (! mutex_trylock(&dev->struct_mutex))
5132                         continue;
5133
5134                 spin_unlock(&shrink_list_lock);
5135
5136                 list_for_each_entry_safe(obj_priv, next_obj,
5137                                          &dev_priv->mm.inactive_list,
5138                                          mm_list) {
5139                         if (nr_to_scan > 0) {
5140                                 i915_gem_object_unbind(&obj_priv->base);
5141                                 nr_to_scan--;
5142                         } else
5143                                 cnt++;
5144                 }
5145
5146                 spin_lock(&shrink_list_lock);
5147                 mutex_unlock(&dev->struct_mutex);
5148
5149                 would_deadlock = 0;
5150         }
5151
5152         if (nr_to_scan) {
5153                 int active = 0;
5154
5155                 /*
5156                  * We are desperate for pages, so as a last resort, wait
5157                  * for the GPU to finish and discard whatever we can.
5158                  * This has a dramatic impact to reduce the number of
5159                  * OOM-killer events whilst running the GPU aggressively.
5160                  */
5161                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5162                         struct drm_device *dev = dev_priv->dev;
5163
5164                         if (!mutex_trylock(&dev->struct_mutex))
5165                                 continue;
5166
5167                         spin_unlock(&shrink_list_lock);
5168
5169                         if (i915_gpu_is_active(dev)) {
5170                                 i915_gpu_idle(dev);
5171                                 active++;
5172                         }
5173
5174                         spin_lock(&shrink_list_lock);
5175                         mutex_unlock(&dev->struct_mutex);
5176                 }
5177
5178                 if (active)
5179                         goto rescan;
5180         }
5181
5182         spin_unlock(&shrink_list_lock);
5183
5184         if (would_deadlock)
5185                 return -1;
5186         else if (cnt > 0)
5187                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5188         else
5189                 return 0;
5190 }
5191
5192 static struct shrinker shrinker = {
5193         .shrink = i915_gem_shrink,
5194         .seeks = DEFAULT_SEEKS,
5195 };
5196
5197 __init void
5198 i915_gem_shrinker_init(void)
5199 {
5200     register_shrinker(&shrinker);
5201 }
5202
5203 __exit void
5204 i915_gem_shrinker_exit(void)
5205 {
5206     unregister_shrinker(&shrinker);
5207 }