drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 struct change_domains {
  40         uint32_t invalidate_domains;
  41         uint32_t flush_domains;
  42         uint32_t flush_rings;
  43 };
  44
  45 static uint32_t i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv);
  46 static uint32_t i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv);
  47
  48 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
  49                                                   bool pipelined);
  50 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  51 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  52 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  53                                              int write);
  54 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  55                                                      uint64_t offset,
  56                                                      uint64_t size);
  57 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  58 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  59                                           bool interruptible);
  60 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  61                                        unsigned alignment,
  62                                        bool map_and_fenceable);
  63 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  64 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  65                                 struct drm_i915_gem_pwrite *args,
  66                                 struct drm_file *file_priv);
  67 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  68
  69 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  70                                     int nr_to_scan,
  71                                     gfp_t gfp_mask);
  72
  73
  74 /* some bookkeeping */
  75 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  76                                   size_t size)
  77 {
  78         dev_priv->mm.object_count++;
  79         dev_priv->mm.object_memory += size;
  80 }
  81
  82 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  83                                      size_t size)
  84 {
  85         dev_priv->mm.object_count--;
  86         dev_priv->mm.object_memory -= size;
  87 }
  88
  89 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
  90                                   struct drm_i915_gem_object *obj)
  91 {
  92         dev_priv->mm.gtt_count++;
  93         dev_priv->mm.gtt_memory += obj->gtt_space->size;
  94         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
  95                 dev_priv->mm.mappable_gtt_used +=
  96                         min_t(size_t, obj->gtt_space->size,
  97                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
  98         }
  99 }
 100
 101 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
 102                                      struct drm_i915_gem_object *obj)
 103 {
 104         dev_priv->mm.gtt_count--;
 105         dev_priv->mm.gtt_memory -= obj->gtt_space->size;
 106         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
 107                 dev_priv->mm.mappable_gtt_used -=
 108                         min_t(size_t, obj->gtt_space->size,
 109                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
 110         }
 111 }
 112
 113 /**
 114  * Update the mappable working set counters. Call _only_ when there is a change
 115  * in one of (pin|fault)_mappable and update *_mappable _before_ calling.
 116  * @mappable: new state the changed mappable flag (either pin_ or fault_).
 117  */
 118 static void
 119 i915_gem_info_update_mappable(struct drm_i915_private *dev_priv,
 120                               struct drm_i915_gem_object *obj,
 121                               bool mappable)
 122 {
 123         if (mappable) {
 124                 if (obj->pin_mappable && obj->fault_mappable)
 125                         /* Combined state was already mappable. */
 126                         return;
 127                 dev_priv->mm.gtt_mappable_count++;
 128                 dev_priv->mm.gtt_mappable_memory += obj->gtt_space->size;
 129         } else {
 130                 if (obj->pin_mappable || obj->fault_mappable)
 131                         /* Combined state still mappable. */
 132                         return;
 133                 dev_priv->mm.gtt_mappable_count--;
 134                 dev_priv->mm.gtt_mappable_memory -= obj->gtt_space->size;
 135         }
 136 }
 137
 138 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
 139                                   struct drm_i915_gem_object *obj,
 140                                   bool mappable)
 141 {
 142         dev_priv->mm.pin_count++;
 143         dev_priv->mm.pin_memory += obj->gtt_space->size;
 144         if (mappable) {
 145                 obj->pin_mappable = true;
 146                 i915_gem_info_update_mappable(dev_priv, obj, true);
 147         }
 148 }
 149
 150 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
 151                                      struct drm_i915_gem_object *obj)
 152 {
 153         dev_priv->mm.pin_count--;
 154         dev_priv->mm.pin_memory -= obj->gtt_space->size;
 155         if (obj->pin_mappable) {
 156                 obj->pin_mappable = false;
 157                 i915_gem_info_update_mappable(dev_priv, obj, false);
 158         }
 159 }
 160
 161 int
 162 i915_gem_check_is_wedged(struct drm_device *dev)
 163 {
 164         struct drm_i915_private *dev_priv = dev->dev_private;
 165         struct completion *x = &dev_priv->error_completion;
 166         unsigned long flags;
 167         int ret;
 168
 169         if (!atomic_read(&dev_priv->mm.wedged))
 170                 return 0;
 171
 172         ret = wait_for_completion_interruptible(x);
 173         if (ret)
 174                 return ret;
 175
 176         /* Success, we reset the GPU! */
 177         if (!atomic_read(&dev_priv->mm.wedged))
 178                 return 0;
 179
 180         /* GPU is hung, bump the completion count to account for
 181          * the token we just consumed so that we never hit zero and
 182          * end up waiting upon a subsequent completion event that
 183          * will never happen.
 184          */
 185         spin_lock_irqsave(&x->wait.lock, flags);
 186         x->done++;
 187         spin_unlock_irqrestore(&x->wait.lock, flags);
 188         return -EIO;
 189 }
 190
 191 static int i915_mutex_lock_interruptible(struct drm_device *dev)
 192 {
 193         struct drm_i915_private *dev_priv = dev->dev_private;
 194         int ret;
 195
 196         ret = i915_gem_check_is_wedged(dev);
 197         if (ret)
 198                 return ret;
 199
 200         ret = mutex_lock_interruptible(&dev->struct_mutex);
 201         if (ret)
 202                 return ret;
 203
 204         if (atomic_read(&dev_priv->mm.wedged)) {
 205                 mutex_unlock(&dev->struct_mutex);
 206                 return -EAGAIN;
 207         }
 208
 209         WARN_ON(i915_verify_lists(dev));
 210         return 0;
 211 }
 212
 213 static inline bool
 214 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
 215 {
 216         return obj_priv->gtt_space &&
 217                 !obj_priv->active &&
 218                 obj_priv->pin_count == 0;
 219 }
 220
 221 int i915_gem_do_init(struct drm_device *dev,
 222                      unsigned long start,
 223                      unsigned long mappable_end,
 224                      unsigned long end)
 225 {
 226         drm_i915_private_t *dev_priv = dev->dev_private;
 227
 228         if (start >= end ||
 229             (start & (PAGE_SIZE - 1)) != 0 ||
 230             (end & (PAGE_SIZE - 1)) != 0) {
 231                 return -EINVAL;
 232         }
 233
 234         drm_mm_init(&dev_priv->mm.gtt_space, start,
 235                     end - start);
 236
 237         dev_priv->mm.gtt_total = end - start;
 238         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 239         dev_priv->mm.gtt_mappable_end = mappable_end;
 240
 241         return 0;
 242 }
 243
 244 int
 245 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 246                     struct drm_file *file_priv)
 247 {
 248         struct drm_i915_gem_init *args = data;
 249         int ret;
 250
 251         mutex_lock(&dev->struct_mutex);
 252         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 253         mutex_unlock(&dev->struct_mutex);
 254
 255         return ret;
 256 }
 257
 258 int
 259 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 260                             struct drm_file *file_priv)
 261 {
 262         struct drm_i915_private *dev_priv = dev->dev_private;
 263         struct drm_i915_gem_get_aperture *args = data;
 264
 265         if (!(dev->driver->driver_features & DRIVER_GEM))
 266                 return -ENODEV;
 267
 268         mutex_lock(&dev->struct_mutex);
 269         args->aper_size = dev_priv->mm.gtt_total;
 270         args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
 271         mutex_unlock(&dev->struct_mutex);
 272
 273         return 0;
 274 }
 275
 276
 277 /**
 278  * Creates a new mm object and returns a handle to it.
 279  */
 280 int
 281 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 282                       struct drm_file *file_priv)
 283 {
 284         struct drm_i915_gem_create *args = data;
 285         struct drm_gem_object *obj;
 286         int ret;
 287         u32 handle;
 288
 289         args->size = roundup(args->size, PAGE_SIZE);
 290
 291         /* Allocate the new object */
 292         obj = i915_gem_alloc_object(dev, args->size);
 293         if (obj == NULL)
 294                 return -ENOMEM;
 295
 296         ret = drm_gem_handle_create(file_priv, obj, &handle);
 297         if (ret) {
 298                 drm_gem_object_release(obj);
 299                 i915_gem_info_remove_obj(dev->dev_private, obj->size);
 300                 kfree(obj);
 301                 return ret;
 302         }
 303
 304         /* drop reference from allocate - handle holds it now */
 305         drm_gem_object_unreference(obj);
 306         trace_i915_gem_object_create(obj);
 307
 308         args->handle = handle;
 309         return 0;
 310 }
 311
 312 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 313 {
 314         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 315         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 316
 317         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 318                 obj_priv->tiling_mode != I915_TILING_NONE;
 319 }
 320
 321 static inline void
 322 slow_shmem_copy(struct page *dst_page,
 323                 int dst_offset,
 324                 struct page *src_page,
 325                 int src_offset,
 326                 int length)
 327 {
 328         char *dst_vaddr, *src_vaddr;
 329
 330         dst_vaddr = kmap(dst_page);
 331         src_vaddr = kmap(src_page);
 332
 333         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 334
 335         kunmap(src_page);
 336         kunmap(dst_page);
 337 }
 338
 339 static inline void
 340 slow_shmem_bit17_copy(struct page *gpu_page,
 341                       int gpu_offset,
 342                       struct page *cpu_page,
 343                       int cpu_offset,
 344                       int length,
 345                       int is_read)
 346 {
 347         char *gpu_vaddr, *cpu_vaddr;
 348
 349         /* Use the unswizzled path if this page isn't affected. */
 350         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 351                 if (is_read)
 352                         return slow_shmem_copy(cpu_page, cpu_offset,
 353                                                gpu_page, gpu_offset, length);
 354                 else
 355                         return slow_shmem_copy(gpu_page, gpu_offset,
 356                                                cpu_page, cpu_offset, length);
 357         }
 358
 359         gpu_vaddr = kmap(gpu_page);
 360         cpu_vaddr = kmap(cpu_page);
 361
 362         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 363          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 364          */
 365         while (length > 0) {
 366                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 367                 int this_length = min(cacheline_end - gpu_offset, length);
 368                 int swizzled_gpu_offset = gpu_offset ^ 64;
 369
 370                 if (is_read) {
 371                         memcpy(cpu_vaddr + cpu_offset,
 372                                gpu_vaddr + swizzled_gpu_offset,
 373                                this_length);
 374                 } else {
 375                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 376                                cpu_vaddr + cpu_offset,
 377                                this_length);
 378                 }
 379                 cpu_offset += this_length;
 380                 gpu_offset += this_length;
 381                 length -= this_length;
 382         }
 383
 384         kunmap(cpu_page);
 385         kunmap(gpu_page);
 386 }
 387
 388 /**
 389  * This is the fast shmem pread path, which attempts to copy_from_user directly
 390  * from the backing pages of the object to the user's address space.  On a
 391  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 392  */
 393 static int
 394 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 395                           struct drm_i915_gem_pread *args,
 396                           struct drm_file *file_priv)
 397 {
 398         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 399         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 400         ssize_t remain;
 401         loff_t offset;
 402         char __user *user_data;
 403         int page_offset, page_length;
 404
 405         user_data = (char __user *) (uintptr_t) args->data_ptr;
 406         remain = args->size;
 407
 408         obj_priv = to_intel_bo(obj);
 409         offset = args->offset;
 410
 411         while (remain > 0) {
 412                 struct page *page;
 413                 char *vaddr;
 414                 int ret;
 415
 416                 /* Operation in this page
 417                  *
 418                  * page_offset = offset within page
 419                  * page_length = bytes to copy for this page
 420                  */
 421                 page_offset = offset & (PAGE_SIZE-1);
 422                 page_length = remain;
 423                 if ((page_offset + remain) > PAGE_SIZE)
 424                         page_length = PAGE_SIZE - page_offset;
 425
 426                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 427                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 428                 if (IS_ERR(page))
 429                         return PTR_ERR(page);
 430
 431                 vaddr = kmap_atomic(page);
 432                 ret = __copy_to_user_inatomic(user_data,
 433                                               vaddr + page_offset,
 434                                               page_length);
 435                 kunmap_atomic(vaddr);
 436
 437                 mark_page_accessed(page);
 438                 page_cache_release(page);
 439                 if (ret)
 440                         return -EFAULT;
 441
 442                 remain -= page_length;
 443                 user_data += page_length;
 444                 offset += page_length;
 445         }
 446
 447         return 0;
 448 }
 449
 450 /**
 451  * This is the fallback shmem pread path, which allocates temporary storage
 452  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 453  * can copy out of the object's backing pages while holding the struct mutex
 454  * and not take page faults.
 455  */
 456 static int
 457 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 458                           struct drm_i915_gem_pread *args,
 459                           struct drm_file *file_priv)
 460 {
 461         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 462         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 463         struct mm_struct *mm = current->mm;
 464         struct page **user_pages;
 465         ssize_t remain;
 466         loff_t offset, pinned_pages, i;
 467         loff_t first_data_page, last_data_page, num_pages;
 468         int shmem_page_offset;
 469         int data_page_index, data_page_offset;
 470         int page_length;
 471         int ret;
 472         uint64_t data_ptr = args->data_ptr;
 473         int do_bit17_swizzling;
 474
 475         remain = args->size;
 476
 477         /* Pin the user pages containing the data.  We can't fault while
 478          * holding the struct mutex, yet we want to hold it while
 479          * dereferencing the user data.
 480          */
 481         first_data_page = data_ptr / PAGE_SIZE;
 482         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 483         num_pages = last_data_page - first_data_page + 1;
 484
 485         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 486         if (user_pages == NULL)
 487                 return -ENOMEM;
 488
 489         mutex_unlock(&dev->struct_mutex);
 490         down_read(&mm->mmap_sem);
 491         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 492                                       num_pages, 1, 0, user_pages, NULL);
 493         up_read(&mm->mmap_sem);
 494         mutex_lock(&dev->struct_mutex);
 495         if (pinned_pages < num_pages) {
 496                 ret = -EFAULT;
 497                 goto out;
 498         }
 499
 500         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 501                                                         args->offset,
 502                                                         args->size);
 503         if (ret)
 504                 goto out;
 505
 506         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 507
 508         obj_priv = to_intel_bo(obj);
 509         offset = args->offset;
 510
 511         while (remain > 0) {
 512                 struct page *page;
 513
 514                 /* Operation in this page
 515                  *
 516                  * shmem_page_offset = offset within page in shmem file
 517                  * data_page_index = page number in get_user_pages return
 518                  * data_page_offset = offset with data_page_index page.
 519                  * page_length = bytes to copy for this page
 520                  */
 521                 shmem_page_offset = offset & ~PAGE_MASK;
 522                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 523                 data_page_offset = data_ptr & ~PAGE_MASK;
 524
 525                 page_length = remain;
 526                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 527                         page_length = PAGE_SIZE - shmem_page_offset;
 528                 if ((data_page_offset + page_length) > PAGE_SIZE)
 529                         page_length = PAGE_SIZE - data_page_offset;
 530
 531                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 532                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 533                 if (IS_ERR(page))
 534                         return PTR_ERR(page);
 535
 536                 if (do_bit17_swizzling) {
 537                         slow_shmem_bit17_copy(page,
 538                                               shmem_page_offset,
 539                                               user_pages[data_page_index],
 540                                               data_page_offset,
 541                                               page_length,
 542                                               1);
 543                 } else {
 544                         slow_shmem_copy(user_pages[data_page_index],
 545                                         data_page_offset,
 546                                         page,
 547                                         shmem_page_offset,
 548                                         page_length);
 549                 }
 550
 551                 mark_page_accessed(page);
 552                 page_cache_release(page);
 553
 554                 remain -= page_length;
 555                 data_ptr += page_length;
 556                 offset += page_length;
 557         }
 558
 559 out:
 560         for (i = 0; i < pinned_pages; i++) {
 561                 SetPageDirty(user_pages[i]);
 562                 mark_page_accessed(user_pages[i]);
 563                 page_cache_release(user_pages[i]);
 564         }
 565         drm_free_large(user_pages);
 566
 567         return ret;
 568 }
 569
 570 /**
 571  * Reads data from the object referenced by handle.
 572  *
 573  * On error, the contents of *data are undefined.
 574  */
 575 int
 576 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 577                      struct drm_file *file_priv)
 578 {
 579         struct drm_i915_gem_pread *args = data;
 580         struct drm_gem_object *obj;
 581         struct drm_i915_gem_object *obj_priv;
 582         int ret = 0;
 583
 584         ret = i915_mutex_lock_interruptible(dev);
 585         if (ret)
 586                 return ret;
 587
 588         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 589         if (obj == NULL) {
 590                 ret = -ENOENT;
 591                 goto unlock;
 592         }
 593         obj_priv = to_intel_bo(obj);
 594
 595         /* Bounds check source.  */
 596         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 597                 ret = -EINVAL;
 598                 goto out;
 599         }
 600
 601         if (args->size == 0)
 602                 goto out;
 603
 604         if (!access_ok(VERIFY_WRITE,
 605                        (char __user *)(uintptr_t)args->data_ptr,
 606                        args->size)) {
 607                 ret = -EFAULT;
 608                 goto out;
 609         }
 610
 611         ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
 612                                        args->size);
 613         if (ret) {
 614                 ret = -EFAULT;
 615                 goto out;
 616         }
 617
 618         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 619                                                         args->offset,
 620                                                         args->size);
 621         if (ret)
 622                 goto out;
 623
 624         ret = -EFAULT;
 625         if (!i915_gem_object_needs_bit17_swizzle(obj))
 626                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 627         if (ret == -EFAULT)
 628                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 629
 630 out:
 631         drm_gem_object_unreference(obj);
 632 unlock:
 633         mutex_unlock(&dev->struct_mutex);
 634         return ret;
 635 }
 636
 637 /* This is the fast write path which cannot handle
 638  * page faults in the source data
 639  */
 640
 641 static inline int
 642 fast_user_write(struct io_mapping *mapping,
 643                 loff_t page_base, int page_offset,
 644                 char __user *user_data,
 645                 int length)
 646 {
 647         char *vaddr_atomic;
 648         unsigned long unwritten;
 649
 650         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 651         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 652                                                       user_data, length);
 653         io_mapping_unmap_atomic(vaddr_atomic);
 654         return unwritten;
 655 }
 656
 657 /* Here's the write path which can sleep for
 658  * page faults
 659  */
 660
 661 static inline void
 662 slow_kernel_write(struct io_mapping *mapping,
 663                   loff_t gtt_base, int gtt_offset,
 664                   struct page *user_page, int user_offset,
 665                   int length)
 666 {
 667         char __iomem *dst_vaddr;
 668         char *src_vaddr;
 669
 670         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 671         src_vaddr = kmap(user_page);
 672
 673         memcpy_toio(dst_vaddr + gtt_offset,
 674                     src_vaddr + user_offset,
 675                     length);
 676
 677         kunmap(user_page);
 678         io_mapping_unmap(dst_vaddr);
 679 }
 680
 681 /**
 682  * This is the fast pwrite path, where we copy the data directly from the
 683  * user into the GTT, uncached.
 684  */
 685 static int
 686 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 687                          struct drm_i915_gem_pwrite *args,
 688                          struct drm_file *file_priv)
 689 {
 690         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 691         drm_i915_private_t *dev_priv = dev->dev_private;
 692         ssize_t remain;
 693         loff_t offset, page_base;
 694         char __user *user_data;
 695         int page_offset, page_length;
 696
 697         user_data = (char __user *) (uintptr_t) args->data_ptr;
 698         remain = args->size;
 699
 700         obj_priv = to_intel_bo(obj);
 701         offset = obj_priv->gtt_offset + args->offset;
 702
 703         while (remain > 0) {
 704                 /* Operation in this page
 705                  *
 706                  * page_base = page offset within aperture
 707                  * page_offset = offset within page
 708                  * page_length = bytes to copy for this page
 709                  */
 710                 page_base = (offset & ~(PAGE_SIZE-1));
 711                 page_offset = offset & (PAGE_SIZE-1);
 712                 page_length = remain;
 713                 if ((page_offset + remain) > PAGE_SIZE)
 714                         page_length = PAGE_SIZE - page_offset;
 715
 716                 /* If we get a fault while copying data, then (presumably) our
 717                  * source page isn't available.  Return the error and we'll
 718                  * retry in the slow path.
 719                  */
 720                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 721                                     page_offset, user_data, page_length))
 722
 723                         return -EFAULT;
 724
 725                 remain -= page_length;
 726                 user_data += page_length;
 727                 offset += page_length;
 728         }
 729
 730         return 0;
 731 }
 732
 733 /**
 734  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 735  * the memory and maps it using kmap_atomic for copying.
 736  *
 737  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 738  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 739  */
 740 static int
 741 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 742                          struct drm_i915_gem_pwrite *args,
 743                          struct drm_file *file_priv)
 744 {
 745         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 746         drm_i915_private_t *dev_priv = dev->dev_private;
 747         ssize_t remain;
 748         loff_t gtt_page_base, offset;
 749         loff_t first_data_page, last_data_page, num_pages;
 750         loff_t pinned_pages, i;
 751         struct page **user_pages;
 752         struct mm_struct *mm = current->mm;
 753         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 754         int ret;
 755         uint64_t data_ptr = args->data_ptr;
 756
 757         remain = args->size;
 758
 759         /* Pin the user pages containing the data.  We can't fault while
 760          * holding the struct mutex, and all of the pwrite implementations
 761          * want to hold it while dereferencing the user data.
 762          */
 763         first_data_page = data_ptr / PAGE_SIZE;
 764         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 765         num_pages = last_data_page - first_data_page + 1;
 766
 767         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 768         if (user_pages == NULL)
 769                 return -ENOMEM;
 770
 771         mutex_unlock(&dev->struct_mutex);
 772         down_read(&mm->mmap_sem);
 773         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 774                                       num_pages, 0, 0, user_pages, NULL);
 775         up_read(&mm->mmap_sem);
 776         mutex_lock(&dev->struct_mutex);
 777         if (pinned_pages < num_pages) {
 778                 ret = -EFAULT;
 779                 goto out_unpin_pages;
 780         }
 781
 782         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 783         if (ret)
 784                 goto out_unpin_pages;
 785
 786         obj_priv = to_intel_bo(obj);
 787         offset = obj_priv->gtt_offset + args->offset;
 788
 789         while (remain > 0) {
 790                 /* Operation in this page
 791                  *
 792                  * gtt_page_base = page offset within aperture
 793                  * gtt_page_offset = offset within page in aperture
 794                  * data_page_index = page number in get_user_pages return
 795                  * data_page_offset = offset with data_page_index page.
 796                  * page_length = bytes to copy for this page
 797                  */
 798                 gtt_page_base = offset & PAGE_MASK;
 799                 gtt_page_offset = offset & ~PAGE_MASK;
 800                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 801                 data_page_offset = data_ptr & ~PAGE_MASK;
 802
 803                 page_length = remain;
 804                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 805                         page_length = PAGE_SIZE - gtt_page_offset;
 806                 if ((data_page_offset + page_length) > PAGE_SIZE)
 807                         page_length = PAGE_SIZE - data_page_offset;
 808
 809                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 810                                   gtt_page_base, gtt_page_offset,
 811                                   user_pages[data_page_index],
 812                                   data_page_offset,
 813                                   page_length);
 814
 815                 remain -= page_length;
 816                 offset += page_length;
 817                 data_ptr += page_length;
 818         }
 819
 820 out_unpin_pages:
 821         for (i = 0; i < pinned_pages; i++)
 822                 page_cache_release(user_pages[i]);
 823         drm_free_large(user_pages);
 824
 825         return ret;
 826 }
 827
 828 /**
 829  * This is the fast shmem pwrite path, which attempts to directly
 830  * copy_from_user into the kmapped pages backing the object.
 831  */
 832 static int
 833 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 834                            struct drm_i915_gem_pwrite *args,
 835                            struct drm_file *file_priv)
 836 {
 837         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 838         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 839         ssize_t remain;
 840         loff_t offset;
 841         char __user *user_data;
 842         int page_offset, page_length;
 843
 844         user_data = (char __user *) (uintptr_t) args->data_ptr;
 845         remain = args->size;
 846
 847         obj_priv = to_intel_bo(obj);
 848         offset = args->offset;
 849         obj_priv->dirty = 1;
 850
 851         while (remain > 0) {
 852                 struct page *page;
 853                 char *vaddr;
 854                 int ret;
 855
 856                 /* Operation in this page
 857                  *
 858                  * page_offset = offset within page
 859                  * page_length = bytes to copy for this page
 860                  */
 861                 page_offset = offset & (PAGE_SIZE-1);
 862                 page_length = remain;
 863                 if ((page_offset + remain) > PAGE_SIZE)
 864                         page_length = PAGE_SIZE - page_offset;
 865
 866                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 867                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 868                 if (IS_ERR(page))
 869                         return PTR_ERR(page);
 870
 871                 vaddr = kmap_atomic(page, KM_USER0);
 872                 ret = __copy_from_user_inatomic(vaddr + page_offset,
 873                                                 user_data,
 874                                                 page_length);
 875                 kunmap_atomic(vaddr, KM_USER0);
 876
 877                 set_page_dirty(page);
 878                 mark_page_accessed(page);
 879                 page_cache_release(page);
 880
 881                 /* If we get a fault while copying data, then (presumably) our
 882                  * source page isn't available.  Return the error and we'll
 883                  * retry in the slow path.
 884                  */
 885                 if (ret)
 886                         return -EFAULT;
 887
 888                 remain -= page_length;
 889                 user_data += page_length;
 890                 offset += page_length;
 891         }
 892
 893         return 0;
 894 }
 895
 896 /**
 897  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 898  * the memory and maps it using kmap_atomic for copying.
 899  *
 900  * This avoids taking mmap_sem for faulting on the user's address while the
 901  * struct_mutex is held.
 902  */
 903 static int
 904 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 905                            struct drm_i915_gem_pwrite *args,
 906                            struct drm_file *file_priv)
 907 {
 908         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 909         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 910         struct mm_struct *mm = current->mm;
 911         struct page **user_pages;
 912         ssize_t remain;
 913         loff_t offset, pinned_pages, i;
 914         loff_t first_data_page, last_data_page, num_pages;
 915         int shmem_page_offset;
 916         int data_page_index,  data_page_offset;
 917         int page_length;
 918         int ret;
 919         uint64_t data_ptr = args->data_ptr;
 920         int do_bit17_swizzling;
 921
 922         remain = args->size;
 923
 924         /* Pin the user pages containing the data.  We can't fault while
 925          * holding the struct mutex, and all of the pwrite implementations
 926          * want to hold it while dereferencing the user data.
 927          */
 928         first_data_page = data_ptr / PAGE_SIZE;
 929         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 930         num_pages = last_data_page - first_data_page + 1;
 931
 932         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 933         if (user_pages == NULL)
 934                 return -ENOMEM;
 935
 936         mutex_unlock(&dev->struct_mutex);
 937         down_read(&mm->mmap_sem);
 938         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 939                                       num_pages, 0, 0, user_pages, NULL);
 940         up_read(&mm->mmap_sem);
 941         mutex_lock(&dev->struct_mutex);
 942         if (pinned_pages < num_pages) {
 943                 ret = -EFAULT;
 944                 goto out;
 945         }
 946
 947         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 948         if (ret)
 949                 goto out;
 950
 951         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 952
 953         obj_priv = to_intel_bo(obj);
 954         offset = args->offset;
 955         obj_priv->dirty = 1;
 956
 957         while (remain > 0) {
 958                 struct page *page;
 959
 960                 /* Operation in this page
 961                  *
 962                  * shmem_page_offset = offset within page in shmem file
 963                  * data_page_index = page number in get_user_pages return
 964                  * data_page_offset = offset with data_page_index page.
 965                  * page_length = bytes to copy for this page
 966                  */
 967                 shmem_page_offset = offset & ~PAGE_MASK;
 968                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 969                 data_page_offset = data_ptr & ~PAGE_MASK;
 970
 971                 page_length = remain;
 972                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 973                         page_length = PAGE_SIZE - shmem_page_offset;
 974                 if ((data_page_offset + page_length) > PAGE_SIZE)
 975                         page_length = PAGE_SIZE - data_page_offset;
 976
 977                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 978                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 979                 if (IS_ERR(page)) {
 980                         ret = PTR_ERR(page);
 981                         goto out;
 982                 }
 983
 984                 if (do_bit17_swizzling) {
 985                         slow_shmem_bit17_copy(page,
 986                                               shmem_page_offset,
 987                                               user_pages[data_page_index],
 988                                               data_page_offset,
 989                                               page_length,
 990                                               0);
 991                 } else {
 992                         slow_shmem_copy(page,
 993                                         shmem_page_offset,
 994                                         user_pages[data_page_index],
 995                                         data_page_offset,
 996                                         page_length);
 997                 }
 998
 999                 set_page_dirty(page);
1000                 mark_page_accessed(page);
1001                 page_cache_release(page);
1002
1003                 remain -= page_length;
1004                 data_ptr += page_length;
1005                 offset += page_length;
1006         }
1007
1008 out:
1009         for (i = 0; i < pinned_pages; i++)
1010                 page_cache_release(user_pages[i]);
1011         drm_free_large(user_pages);
1012
1013         return ret;
1014 }
1015
1016 /**
1017  * Writes data to the object referenced by handle.
1018  *
1019  * On error, the contents of the buffer that were to be modified are undefined.
1020  */
1021 int
1022 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1023                       struct drm_file *file)
1024 {
1025         struct drm_i915_gem_pwrite *args = data;
1026         struct drm_gem_object *obj;
1027         struct drm_i915_gem_object *obj_priv;
1028         int ret = 0;
1029
1030         ret = i915_mutex_lock_interruptible(dev);
1031         if (ret)
1032                 return ret;
1033
1034         obj = drm_gem_object_lookup(dev, file, args->handle);
1035         if (obj == NULL) {
1036                 ret = -ENOENT;
1037                 goto unlock;
1038         }
1039         obj_priv = to_intel_bo(obj);
1040
1041
1042         /* Bounds check destination. */
1043         if (args->offset > obj->size || args->size > obj->size - args->offset) {
1044                 ret = -EINVAL;
1045                 goto out;
1046         }
1047
1048         if (args->size == 0)
1049                 goto out;
1050
1051         if (!access_ok(VERIFY_READ,
1052                        (char __user *)(uintptr_t)args->data_ptr,
1053                        args->size)) {
1054                 ret = -EFAULT;
1055                 goto out;
1056         }
1057
1058         ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1059                                       args->size);
1060         if (ret) {
1061                 ret = -EFAULT;
1062                 goto out;
1063         }
1064
1065         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1066          * it would end up going through the fenced access, and we'll get
1067          * different detiling behavior between reading and writing.
1068          * pread/pwrite currently are reading and writing from the CPU
1069          * perspective, requiring manual detiling by the client.
1070          */
1071         if (obj_priv->phys_obj)
1072                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1073         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1074                  obj_priv->gtt_space &&
1075                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
1076                 ret = i915_gem_object_pin(obj, 0, true);
1077                 if (ret)
1078                         goto out;
1079
1080                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1081                 if (ret)
1082                         goto out_unpin;
1083
1084                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1085                 if (ret == -EFAULT)
1086                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1087
1088 out_unpin:
1089                 i915_gem_object_unpin(obj);
1090         } else {
1091                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1092                 if (ret)
1093                         goto out;
1094
1095                 ret = -EFAULT;
1096                 if (!i915_gem_object_needs_bit17_swizzle(obj))
1097                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1098                 if (ret == -EFAULT)
1099                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1100         }
1101
1102 out:
1103         drm_gem_object_unreference(obj);
1104 unlock:
1105         mutex_unlock(&dev->struct_mutex);
1106         return ret;
1107 }
1108
1109 /**
1110  * Called when user space prepares to use an object with the CPU, either
1111  * through the mmap ioctl's mapping or a GTT mapping.
1112  */
1113 int
1114 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1115                           struct drm_file *file_priv)
1116 {
1117         struct drm_i915_private *dev_priv = dev->dev_private;
1118         struct drm_i915_gem_set_domain *args = data;
1119         struct drm_gem_object *obj;
1120         struct drm_i915_gem_object *obj_priv;
1121         uint32_t read_domains = args->read_domains;
1122         uint32_t write_domain = args->write_domain;
1123         int ret;
1124
1125         if (!(dev->driver->driver_features & DRIVER_GEM))
1126                 return -ENODEV;
1127
1128         /* Only handle setting domains to types used by the CPU. */
1129         if (write_domain & I915_GEM_GPU_DOMAINS)
1130                 return -EINVAL;
1131
1132         if (read_domains & I915_GEM_GPU_DOMAINS)
1133                 return -EINVAL;
1134
1135         /* Having something in the write domain implies it's in the read
1136          * domain, and only that read domain.  Enforce that in the request.
1137          */
1138         if (write_domain != 0 && read_domains != write_domain)
1139                 return -EINVAL;
1140
1141         ret = i915_mutex_lock_interruptible(dev);
1142         if (ret)
1143                 return ret;
1144
1145         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1146         if (obj == NULL) {
1147                 ret = -ENOENT;
1148                 goto unlock;
1149         }
1150         obj_priv = to_intel_bo(obj);
1151
1152         intel_mark_busy(dev, obj);
1153
1154         if (read_domains & I915_GEM_DOMAIN_GTT) {
1155                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1156
1157                 /* Update the LRU on the fence for the CPU access that's
1158                  * about to occur.
1159                  */
1160                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1161                         struct drm_i915_fence_reg *reg =
1162                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1163                         list_move_tail(&reg->lru_list,
1164                                        &dev_priv->mm.fence_list);
1165                 }
1166
1167                 /* Silently promote "you're not bound, there was nothing to do"
1168                  * to success, since the client was just asking us to
1169                  * make sure everything was done.
1170                  */
1171                 if (ret == -EINVAL)
1172                         ret = 0;
1173         } else {
1174                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1175         }
1176
1177         /* Maintain LRU order of "inactive" objects */
1178         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1179                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1180
1181         drm_gem_object_unreference(obj);
1182 unlock:
1183         mutex_unlock(&dev->struct_mutex);
1184         return ret;
1185 }
1186
1187 /**
1188  * Called when user space has done writes to this buffer
1189  */
1190 int
1191 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1192                       struct drm_file *file_priv)
1193 {
1194         struct drm_i915_gem_sw_finish *args = data;
1195         struct drm_gem_object *obj;
1196         int ret = 0;
1197
1198         if (!(dev->driver->driver_features & DRIVER_GEM))
1199                 return -ENODEV;
1200
1201         ret = i915_mutex_lock_interruptible(dev);
1202         if (ret)
1203                 return ret;
1204
1205         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1206         if (obj == NULL) {
1207                 ret = -ENOENT;
1208                 goto unlock;
1209         }
1210
1211         /* Pinned buffers may be scanout, so flush the cache */
1212         if (to_intel_bo(obj)->pin_count)
1213                 i915_gem_object_flush_cpu_write_domain(obj);
1214
1215         drm_gem_object_unreference(obj);
1216 unlock:
1217         mutex_unlock(&dev->struct_mutex);
1218         return ret;
1219 }
1220
1221 /**
1222  * Maps the contents of an object, returning the address it is mapped
1223  * into.
1224  *
1225  * While the mapping holds a reference on the contents of the object, it doesn't
1226  * imply a ref on the object itself.
1227  */
1228 int
1229 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1230                    struct drm_file *file_priv)
1231 {
1232         struct drm_i915_private *dev_priv = dev->dev_private;
1233         struct drm_i915_gem_mmap *args = data;
1234         struct drm_gem_object *obj;
1235         loff_t offset;
1236         unsigned long addr;
1237
1238         if (!(dev->driver->driver_features & DRIVER_GEM))
1239                 return -ENODEV;
1240
1241         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1242         if (obj == NULL)
1243                 return -ENOENT;
1244
1245         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1246                 drm_gem_object_unreference_unlocked(obj);
1247                 return -E2BIG;
1248         }
1249
1250         offset = args->offset;
1251
1252         down_write(&current->mm->mmap_sem);
1253         addr = do_mmap(obj->filp, 0, args->size,
1254                        PROT_READ | PROT_WRITE, MAP_SHARED,
1255                        args->offset);
1256         up_write(&current->mm->mmap_sem);
1257         drm_gem_object_unreference_unlocked(obj);
1258         if (IS_ERR((void *)addr))
1259                 return addr;
1260
1261         args->addr_ptr = (uint64_t) addr;
1262
1263         return 0;
1264 }
1265
1266 /**
1267  * i915_gem_fault - fault a page into the GTT
1268  * vma: VMA in question
1269  * vmf: fault info
1270  *
1271  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1272  * from userspace.  The fault handler takes care of binding the object to
1273  * the GTT (if needed), allocating and programming a fence register (again,
1274  * only if needed based on whether the old reg is still valid or the object
1275  * is tiled) and inserting a new PTE into the faulting process.
1276  *
1277  * Note that the faulting process may involve evicting existing objects
1278  * from the GTT and/or fence registers to make room.  So performance may
1279  * suffer if the GTT working set is large or there are few fence registers
1280  * left.
1281  */
1282 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1283 {
1284         struct drm_gem_object *obj = vma->vm_private_data;
1285         struct drm_device *dev = obj->dev;
1286         drm_i915_private_t *dev_priv = dev->dev_private;
1287         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1288         pgoff_t page_offset;
1289         unsigned long pfn;
1290         int ret = 0;
1291         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1292
1293         /* We don't use vmf->pgoff since that has the fake offset */
1294         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1295                 PAGE_SHIFT;
1296
1297         /* Now bind it into the GTT if needed */
1298         mutex_lock(&dev->struct_mutex);
1299         BUG_ON(obj_priv->pin_count && !obj_priv->pin_mappable);
1300
1301         if (obj_priv->gtt_space) {
1302                 if (!obj_priv->map_and_fenceable) {
1303                         ret = i915_gem_object_unbind(obj);
1304                         if (ret)
1305                                 goto unlock;
1306                 }
1307         }
1308
1309         if (!obj_priv->gtt_space) {
1310                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1311                 if (ret)
1312                         goto unlock;
1313         }
1314
1315         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1316         if (ret)
1317                 goto unlock;
1318
1319         if (!obj_priv->fault_mappable) {
1320                 obj_priv->fault_mappable = true;
1321                 i915_gem_info_update_mappable(dev_priv, obj_priv, true);
1322         }
1323
1324         /* Need a new fence register? */
1325         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1326                 ret = i915_gem_object_get_fence_reg(obj, true);
1327                 if (ret)
1328                         goto unlock;
1329         }
1330
1331         if (i915_gem_object_is_inactive(obj_priv))
1332                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1333
1334         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1335                 page_offset;
1336
1337         /* Finally, remap it using the new GTT offset */
1338         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1339 unlock:
1340         mutex_unlock(&dev->struct_mutex);
1341
1342         switch (ret) {
1343         case -EAGAIN:
1344                 set_need_resched();
1345         case 0:
1346         case -ERESTARTSYS:
1347                 return VM_FAULT_NOPAGE;
1348         case -ENOMEM:
1349                 return VM_FAULT_OOM;
1350         default:
1351                 return VM_FAULT_SIGBUS;
1352         }
1353 }
1354
1355 /**
1356  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1357  * @obj: obj in question
1358  *
1359  * GEM memory mapping works by handing back to userspace a fake mmap offset
1360  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1361  * up the object based on the offset and sets up the various memory mapping
1362  * structures.
1363  *
1364  * This routine allocates and attaches a fake offset for @obj.
1365  */
1366 static int
1367 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1368 {
1369         struct drm_device *dev = obj->dev;
1370         struct drm_gem_mm *mm = dev->mm_private;
1371         struct drm_map_list *list;
1372         struct drm_local_map *map;
1373         int ret = 0;
1374
1375         /* Set the object up for mmap'ing */
1376         list = &obj->map_list;
1377         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1378         if (!list->map)
1379                 return -ENOMEM;
1380
1381         map = list->map;
1382         map->type = _DRM_GEM;
1383         map->size = obj->size;
1384         map->handle = obj;
1385
1386         /* Get a DRM GEM mmap offset allocated... */
1387         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1388                                                     obj->size / PAGE_SIZE, 0, 0);
1389         if (!list->file_offset_node) {
1390                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1391                 ret = -ENOSPC;
1392                 goto out_free_list;
1393         }
1394
1395         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1396                                                   obj->size / PAGE_SIZE, 0);
1397         if (!list->file_offset_node) {
1398                 ret = -ENOMEM;
1399                 goto out_free_list;
1400         }
1401
1402         list->hash.key = list->file_offset_node->start;
1403         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1404         if (ret) {
1405                 DRM_ERROR("failed to add to map hash\n");
1406                 goto out_free_mm;
1407         }
1408
1409         return 0;
1410
1411 out_free_mm:
1412         drm_mm_put_block(list->file_offset_node);
1413 out_free_list:
1414         kfree(list->map);
1415         list->map = NULL;
1416
1417         return ret;
1418 }
1419
1420 /**
1421  * i915_gem_release_mmap - remove physical page mappings
1422  * @obj: obj in question
1423  *
1424  * Preserve the reservation of the mmapping with the DRM core code, but
1425  * relinquish ownership of the pages back to the system.
1426  *
1427  * It is vital that we remove the page mapping if we have mapped a tiled
1428  * object through the GTT and then lose the fence register due to
1429  * resource pressure. Similarly if the object has been moved out of the
1430  * aperture, than pages mapped into userspace must be revoked. Removing the
1431  * mapping will then trigger a page fault on the next user access, allowing
1432  * fixup by i915_gem_fault().
1433  */
1434 void
1435 i915_gem_release_mmap(struct drm_gem_object *obj)
1436 {
1437         struct drm_device *dev = obj->dev;
1438         struct drm_i915_private *dev_priv = dev->dev_private;
1439         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1440
1441         if (unlikely(obj->map_list.map && dev->dev_mapping))
1442                 unmap_mapping_range(dev->dev_mapping,
1443                                     (loff_t)obj->map_list.hash.key<<PAGE_SHIFT,
1444                                     obj->size, 1);
1445
1446         if (obj_priv->fault_mappable) {
1447                 obj_priv->fault_mappable = false;
1448                 i915_gem_info_update_mappable(dev_priv, obj_priv, false);
1449         }
1450 }
1451
1452 static void
1453 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1454 {
1455         struct drm_device *dev = obj->dev;
1456         struct drm_gem_mm *mm = dev->mm_private;
1457         struct drm_map_list *list = &obj->map_list;
1458
1459         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1460         drm_mm_put_block(list->file_offset_node);
1461         kfree(list->map);
1462         list->map = NULL;
1463 }
1464
1465 /**
1466  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1467  * @obj: object to check
1468  *
1469  * Return the required GTT alignment for an object, taking into account
1470  * potential fence register mapping if needed.
1471  */
1472 static uint32_t
1473 i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv)
1474 {
1475         struct drm_device *dev = obj_priv->base.dev;
1476
1477         /*
1478          * Minimum alignment is 4k (GTT page size), but might be greater
1479          * if a fence register is needed for the object.
1480          */
1481         if (INTEL_INFO(dev)->gen >= 4 ||
1482             obj_priv->tiling_mode == I915_TILING_NONE)
1483                 return 4096;
1484
1485         /*
1486          * Previous chips need to be aligned to the size of the smallest
1487          * fence register that can contain the object.
1488          */
1489         return i915_gem_get_gtt_size(obj_priv);
1490 }
1491
1492 static uint32_t
1493 i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv)
1494 {
1495         struct drm_device *dev = obj_priv->base.dev;
1496         uint32_t size;
1497
1498         /*
1499          * Minimum alignment is 4k (GTT page size), but might be greater
1500          * if a fence register is needed for the object.
1501          */
1502         if (INTEL_INFO(dev)->gen >= 4)
1503                 return obj_priv->base.size;
1504
1505         /*
1506          * Previous chips need to be aligned to the size of the smallest
1507          * fence register that can contain the object.
1508          */
1509         if (INTEL_INFO(dev)->gen == 3)
1510                 size = 1024*1024;
1511         else
1512                 size = 512*1024;
1513
1514         while (size < obj_priv->base.size)
1515                 size <<= 1;
1516
1517         return size;
1518 }
1519
1520 /**
1521  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1522  * @dev: DRM device
1523  * @data: GTT mapping ioctl data
1524  * @file_priv: GEM object info
1525  *
1526  * Simply returns the fake offset to userspace so it can mmap it.
1527  * The mmap call will end up in drm_gem_mmap(), which will set things
1528  * up so we can get faults in the handler above.
1529  *
1530  * The fault handler will take care of binding the object into the GTT
1531  * (since it may have been evicted to make room for something), allocating
1532  * a fence register, and mapping the appropriate aperture address into
1533  * userspace.
1534  */
1535 int
1536 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1537                         struct drm_file *file_priv)
1538 {
1539         struct drm_i915_private *dev_priv = dev->dev_private;
1540         struct drm_i915_gem_mmap_gtt *args = data;
1541         struct drm_gem_object *obj;
1542         struct drm_i915_gem_object *obj_priv;
1543         int ret;
1544
1545         if (!(dev->driver->driver_features & DRIVER_GEM))
1546                 return -ENODEV;
1547
1548         ret = i915_mutex_lock_interruptible(dev);
1549         if (ret)
1550                 return ret;
1551
1552         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1553         if (obj == NULL) {
1554                 ret = -ENOENT;
1555                 goto unlock;
1556         }
1557         obj_priv = to_intel_bo(obj);
1558
1559         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1560                 ret = -E2BIG;
1561                 goto unlock;
1562         }
1563
1564         if (obj_priv->madv != I915_MADV_WILLNEED) {
1565                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1566                 ret = -EINVAL;
1567                 goto out;
1568         }
1569
1570         if (!obj->map_list.map) {
1571                 ret = i915_gem_create_mmap_offset(obj);
1572                 if (ret)
1573                         goto out;
1574         }
1575
1576         args->offset = (u64)obj->map_list.hash.key << PAGE_SHIFT;
1577
1578 out:
1579         drm_gem_object_unreference(obj);
1580 unlock:
1581         mutex_unlock(&dev->struct_mutex);
1582         return ret;
1583 }
1584
1585 static int
1586 i915_gem_object_get_pages_gtt(struct drm_gem_object *obj,
1587                               gfp_t gfpmask)
1588 {
1589         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1590         int page_count, i;
1591         struct address_space *mapping;
1592         struct inode *inode;
1593         struct page *page;
1594
1595         /* Get the list of pages out of our struct file.  They'll be pinned
1596          * at this point until we release them.
1597          */
1598         page_count = obj->size / PAGE_SIZE;
1599         BUG_ON(obj_priv->pages != NULL);
1600         obj_priv->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1601         if (obj_priv->pages == NULL)
1602                 return -ENOMEM;
1603
1604         inode = obj->filp->f_path.dentry->d_inode;
1605         mapping = inode->i_mapping;
1606         for (i = 0; i < page_count; i++) {
1607                 page = read_cache_page_gfp(mapping, i,
1608                                            GFP_HIGHUSER |
1609                                            __GFP_COLD |
1610                                            __GFP_RECLAIMABLE |
1611                                            gfpmask);
1612                 if (IS_ERR(page))
1613                         goto err_pages;
1614
1615                 obj_priv->pages[i] = page;
1616         }
1617
1618         if (obj_priv->tiling_mode != I915_TILING_NONE)
1619                 i915_gem_object_do_bit_17_swizzle(obj);
1620
1621         return 0;
1622
1623 err_pages:
1624         while (i--)
1625                 page_cache_release(obj_priv->pages[i]);
1626
1627         drm_free_large(obj_priv->pages);
1628         obj_priv->pages = NULL;
1629         return PTR_ERR(page);
1630 }
1631
1632 static void
1633 i915_gem_object_put_pages_gtt(struct drm_gem_object *obj)
1634 {
1635         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1636         int page_count = obj->size / PAGE_SIZE;
1637         int i;
1638
1639         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1640
1641         if (obj_priv->tiling_mode != I915_TILING_NONE)
1642                 i915_gem_object_save_bit_17_swizzle(obj);
1643
1644         if (obj_priv->madv == I915_MADV_DONTNEED)
1645                 obj_priv->dirty = 0;
1646
1647         for (i = 0; i < page_count; i++) {
1648                 if (obj_priv->dirty)
1649                         set_page_dirty(obj_priv->pages[i]);
1650
1651                 if (obj_priv->madv == I915_MADV_WILLNEED)
1652                         mark_page_accessed(obj_priv->pages[i]);
1653
1654                 page_cache_release(obj_priv->pages[i]);
1655         }
1656         obj_priv->dirty = 0;
1657
1658         drm_free_large(obj_priv->pages);
1659         obj_priv->pages = NULL;
1660 }
1661
1662 static uint32_t
1663 i915_gem_next_request_seqno(struct drm_device *dev,
1664                             struct intel_ring_buffer *ring)
1665 {
1666         drm_i915_private_t *dev_priv = dev->dev_private;
1667
1668         ring->outstanding_lazy_request = true;
1669         return dev_priv->next_seqno;
1670 }
1671
1672 static void
1673 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1674                                struct intel_ring_buffer *ring)
1675 {
1676         struct drm_device *dev = obj->dev;
1677         struct drm_i915_private *dev_priv = dev->dev_private;
1678         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1679         uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1680
1681         BUG_ON(ring == NULL);
1682         obj_priv->ring = ring;
1683
1684         /* Add a reference if we're newly entering the active list. */
1685         if (!obj_priv->active) {
1686                 drm_gem_object_reference(obj);
1687                 obj_priv->active = 1;
1688         }
1689
1690         /* Move from whatever list we were on to the tail of execution. */
1691         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1692         list_move_tail(&obj_priv->ring_list, &ring->active_list);
1693         obj_priv->last_rendering_seqno = seqno;
1694 }
1695
1696 static void
1697 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1698 {
1699         struct drm_device *dev = obj->dev;
1700         drm_i915_private_t *dev_priv = dev->dev_private;
1701         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1702
1703         BUG_ON(!obj_priv->active);
1704         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1705         list_del_init(&obj_priv->ring_list);
1706         obj_priv->last_rendering_seqno = 0;
1707 }
1708
1709 /* Immediately discard the backing storage */
1710 static void
1711 i915_gem_object_truncate(struct drm_gem_object *obj)
1712 {
1713         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1714         struct inode *inode;
1715
1716         /* Our goal here is to return as much of the memory as
1717          * is possible back to the system as we are called from OOM.
1718          * To do this we must instruct the shmfs to drop all of its
1719          * backing pages, *now*. Here we mirror the actions taken
1720          * when by shmem_delete_inode() to release the backing store.
1721          */
1722         inode = obj->filp->f_path.dentry->d_inode;
1723         truncate_inode_pages(inode->i_mapping, 0);
1724         if (inode->i_op->truncate_range)
1725                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1726
1727         obj_priv->madv = __I915_MADV_PURGED;
1728 }
1729
1730 static inline int
1731 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1732 {
1733         return obj_priv->madv == I915_MADV_DONTNEED;
1734 }
1735
1736 static void
1737 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1738 {
1739         struct drm_device *dev = obj->dev;
1740         drm_i915_private_t *dev_priv = dev->dev_private;
1741         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1742
1743         if (obj_priv->pin_count != 0)
1744                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1745         else
1746                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1747         list_del_init(&obj_priv->ring_list);
1748
1749         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1750
1751         obj_priv->last_rendering_seqno = 0;
1752         obj_priv->ring = NULL;
1753         if (obj_priv->active) {
1754                 obj_priv->active = 0;
1755                 drm_gem_object_unreference(obj);
1756         }
1757         WARN_ON(i915_verify_lists(dev));
1758 }
1759
1760 static void
1761 i915_gem_process_flushing_list(struct drm_device *dev,
1762                                uint32_t flush_domains,
1763                                struct intel_ring_buffer *ring)
1764 {
1765         drm_i915_private_t *dev_priv = dev->dev_private;
1766         struct drm_i915_gem_object *obj_priv, *next;
1767
1768         list_for_each_entry_safe(obj_priv, next,
1769                                  &ring->gpu_write_list,
1770                                  gpu_write_list) {
1771                 struct drm_gem_object *obj = &obj_priv->base;
1772
1773                 if (obj->write_domain & flush_domains) {
1774                         uint32_t old_write_domain = obj->write_domain;
1775
1776                         obj->write_domain = 0;
1777                         list_del_init(&obj_priv->gpu_write_list);
1778                         i915_gem_object_move_to_active(obj, ring);
1779
1780                         /* update the fence lru list */
1781                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1782                                 struct drm_i915_fence_reg *reg =
1783                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1784                                 list_move_tail(&reg->lru_list,
1785                                                 &dev_priv->mm.fence_list);
1786                         }
1787
1788                         trace_i915_gem_object_change_domain(obj,
1789                                                             obj->read_domains,
1790                                                             old_write_domain);
1791                 }
1792         }
1793 }
1794
1795 int
1796 i915_add_request(struct drm_device *dev,
1797                  struct drm_file *file,
1798                  struct drm_i915_gem_request *request,
1799                  struct intel_ring_buffer *ring)
1800 {
1801         drm_i915_private_t *dev_priv = dev->dev_private;
1802         struct drm_i915_file_private *file_priv = NULL;
1803         uint32_t seqno;
1804         int was_empty;
1805         int ret;
1806
1807         BUG_ON(request == NULL);
1808
1809         if (file != NULL)
1810                 file_priv = file->driver_priv;
1811
1812         ret = ring->add_request(ring, &seqno);
1813         if (ret)
1814             return ret;
1815
1816         ring->outstanding_lazy_request = false;
1817
1818         request->seqno = seqno;
1819         request->ring = ring;
1820         request->emitted_jiffies = jiffies;
1821         was_empty = list_empty(&ring->request_list);
1822         list_add_tail(&request->list, &ring->request_list);
1823
1824         if (file_priv) {
1825                 spin_lock(&file_priv->mm.lock);
1826                 request->file_priv = file_priv;
1827                 list_add_tail(&request->client_list,
1828                               &file_priv->mm.request_list);
1829                 spin_unlock(&file_priv->mm.lock);
1830         }
1831
1832         if (!dev_priv->mm.suspended) {
1833                 mod_timer(&dev_priv->hangcheck_timer,
1834                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1835                 if (was_empty)
1836                         queue_delayed_work(dev_priv->wq,
1837                                            &dev_priv->mm.retire_work, HZ);
1838         }
1839         return 0;
1840 }
1841
1842 /**
1843  * Command execution barrier
1844  *
1845  * Ensures that all commands in the ring are finished
1846  * before signalling the CPU
1847  */
1848 static void
1849 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1850 {
1851         uint32_t flush_domains = 0;
1852
1853         /* The sampler always gets flushed on i965 (sigh) */
1854         if (INTEL_INFO(dev)->gen >= 4)
1855                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1856
1857         ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
1858 }
1859
1860 static inline void
1861 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1862 {
1863         struct drm_i915_file_private *file_priv = request->file_priv;
1864
1865         if (!file_priv)
1866                 return;
1867
1868         spin_lock(&file_priv->mm.lock);
1869         list_del(&request->client_list);
1870         request->file_priv = NULL;
1871         spin_unlock(&file_priv->mm.lock);
1872 }
1873
1874 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1875                                       struct intel_ring_buffer *ring)
1876 {
1877         while (!list_empty(&ring->request_list)) {
1878                 struct drm_i915_gem_request *request;
1879
1880                 request = list_first_entry(&ring->request_list,
1881                                            struct drm_i915_gem_request,
1882                                            list);
1883
1884                 list_del(&request->list);
1885                 i915_gem_request_remove_from_client(request);
1886                 kfree(request);
1887         }
1888
1889         while (!list_empty(&ring->active_list)) {
1890                 struct drm_i915_gem_object *obj_priv;
1891
1892                 obj_priv = list_first_entry(&ring->active_list,
1893                                             struct drm_i915_gem_object,
1894                                             ring_list);
1895
1896                 obj_priv->base.write_domain = 0;
1897                 list_del_init(&obj_priv->gpu_write_list);
1898                 i915_gem_object_move_to_inactive(&obj_priv->base);
1899         }
1900 }
1901
1902 void i915_gem_reset(struct drm_device *dev)
1903 {
1904         struct drm_i915_private *dev_priv = dev->dev_private;
1905         struct drm_i915_gem_object *obj_priv;
1906         int i;
1907
1908         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1909         i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1910         i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1911
1912         /* Remove anything from the flushing lists. The GPU cache is likely
1913          * to be lost on reset along with the data, so simply move the
1914          * lost bo to the inactive list.
1915          */
1916         while (!list_empty(&dev_priv->mm.flushing_list)) {
1917                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1918                                             struct drm_i915_gem_object,
1919                                             mm_list);
1920
1921                 obj_priv->base.write_domain = 0;
1922                 list_del_init(&obj_priv->gpu_write_list);
1923                 i915_gem_object_move_to_inactive(&obj_priv->base);
1924         }
1925
1926         /* Move everything out of the GPU domains to ensure we do any
1927          * necessary invalidation upon reuse.
1928          */
1929         list_for_each_entry(obj_priv,
1930                             &dev_priv->mm.inactive_list,
1931                             mm_list)
1932         {
1933                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1934         }
1935
1936         /* The fence registers are invalidated so clear them out */
1937         for (i = 0; i < 16; i++) {
1938                 struct drm_i915_fence_reg *reg;
1939
1940                 reg = &dev_priv->fence_regs[i];
1941                 if (!reg->obj)
1942                         continue;
1943
1944                 i915_gem_clear_fence_reg(reg->obj);
1945         }
1946 }
1947
1948 /**
1949  * This function clears the request list as sequence numbers are passed.
1950  */
1951 static void
1952 i915_gem_retire_requests_ring(struct drm_device *dev,
1953                               struct intel_ring_buffer *ring)
1954 {
1955         drm_i915_private_t *dev_priv = dev->dev_private;
1956         uint32_t seqno;
1957
1958         if (!ring->status_page.page_addr ||
1959             list_empty(&ring->request_list))
1960                 return;
1961
1962         WARN_ON(i915_verify_lists(dev));
1963
1964         seqno = ring->get_seqno(ring);
1965         while (!list_empty(&ring->request_list)) {
1966                 struct drm_i915_gem_request *request;
1967
1968                 request = list_first_entry(&ring->request_list,
1969                                            struct drm_i915_gem_request,
1970                                            list);
1971
1972                 if (!i915_seqno_passed(seqno, request->seqno))
1973                         break;
1974
1975                 trace_i915_gem_request_retire(dev, request->seqno);
1976
1977                 list_del(&request->list);
1978                 i915_gem_request_remove_from_client(request);
1979                 kfree(request);
1980         }
1981
1982         /* Move any buffers on the active list that are no longer referenced
1983          * by the ringbuffer to the flushing/inactive lists as appropriate.
1984          */
1985         while (!list_empty(&ring->active_list)) {
1986                 struct drm_gem_object *obj;
1987                 struct drm_i915_gem_object *obj_priv;
1988
1989                 obj_priv = list_first_entry(&ring->active_list,
1990                                             struct drm_i915_gem_object,
1991                                             ring_list);
1992
1993                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
1994                         break;
1995
1996                 obj = &obj_priv->base;
1997                 if (obj->write_domain != 0)
1998                         i915_gem_object_move_to_flushing(obj);
1999                 else
2000                         i915_gem_object_move_to_inactive(obj);
2001         }
2002
2003         if (unlikely (dev_priv->trace_irq_seqno &&
2004                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
2005                 ring->user_irq_put(ring);
2006                 dev_priv->trace_irq_seqno = 0;
2007         }
2008
2009         WARN_ON(i915_verify_lists(dev));
2010 }
2011
2012 void
2013 i915_gem_retire_requests(struct drm_device *dev)
2014 {
2015         drm_i915_private_t *dev_priv = dev->dev_private;
2016
2017         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2018             struct drm_i915_gem_object *obj_priv, *tmp;
2019
2020             /* We must be careful that during unbind() we do not
2021              * accidentally infinitely recurse into retire requests.
2022              * Currently:
2023              *   retire -> free -> unbind -> wait -> retire_ring
2024              */
2025             list_for_each_entry_safe(obj_priv, tmp,
2026                                      &dev_priv->mm.deferred_free_list,
2027                                      mm_list)
2028                     i915_gem_free_object_tail(&obj_priv->base);
2029         }
2030
2031         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
2032         i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
2033         i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
2034 }
2035
2036 static void
2037 i915_gem_retire_work_handler(struct work_struct *work)
2038 {
2039         drm_i915_private_t *dev_priv;
2040         struct drm_device *dev;
2041
2042         dev_priv = container_of(work, drm_i915_private_t,
2043                                 mm.retire_work.work);
2044         dev = dev_priv->dev;
2045
2046         /* Come back later if the device is busy... */
2047         if (!mutex_trylock(&dev->struct_mutex)) {
2048                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2049                 return;
2050         }
2051
2052         i915_gem_retire_requests(dev);
2053
2054         if (!dev_priv->mm.suspended &&
2055                 (!list_empty(&dev_priv->render_ring.request_list) ||
2056                  !list_empty(&dev_priv->bsd_ring.request_list) ||
2057                  !list_empty(&dev_priv->blt_ring.request_list)))
2058                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2059         mutex_unlock(&dev->struct_mutex);
2060 }
2061
2062 int
2063 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
2064                      bool interruptible, struct intel_ring_buffer *ring)
2065 {
2066         drm_i915_private_t *dev_priv = dev->dev_private;
2067         u32 ier;
2068         int ret = 0;
2069
2070         BUG_ON(seqno == 0);
2071
2072         if (atomic_read(&dev_priv->mm.wedged))
2073                 return -EAGAIN;
2074
2075         if (ring->outstanding_lazy_request) {
2076                 struct drm_i915_gem_request *request;
2077
2078                 request = kzalloc(sizeof(*request), GFP_KERNEL);
2079                 if (request == NULL)
2080                         return -ENOMEM;
2081
2082                 ret = i915_add_request(dev, NULL, request, ring);
2083                 if (ret) {
2084                         kfree(request);
2085                         return ret;
2086                 }
2087
2088                 seqno = request->seqno;
2089         }
2090         BUG_ON(seqno == dev_priv->next_seqno);
2091
2092         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2093                 if (HAS_PCH_SPLIT(dev))
2094                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2095                 else
2096                         ier = I915_READ(IER);
2097                 if (!ier) {
2098                         DRM_ERROR("something (likely vbetool) disabled "
2099                                   "interrupts, re-enabling\n");
2100                         i915_driver_irq_preinstall(dev);
2101                         i915_driver_irq_postinstall(dev);
2102                 }
2103
2104                 trace_i915_gem_request_wait_begin(dev, seqno);
2105
2106                 ring->waiting_seqno = seqno;
2107                 ring->user_irq_get(ring);
2108                 if (interruptible)
2109                         ret = wait_event_interruptible(ring->irq_queue,
2110                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2111                                 || atomic_read(&dev_priv->mm.wedged));
2112                 else
2113                         wait_event(ring->irq_queue,
2114                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2115                                 || atomic_read(&dev_priv->mm.wedged));
2116
2117                 ring->user_irq_put(ring);
2118                 ring->waiting_seqno = 0;
2119
2120                 trace_i915_gem_request_wait_end(dev, seqno);
2121         }
2122         if (atomic_read(&dev_priv->mm.wedged))
2123                 ret = -EAGAIN;
2124
2125         if (ret && ret != -ERESTARTSYS)
2126                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2127                           __func__, ret, seqno, ring->get_seqno(ring),
2128                           dev_priv->next_seqno);
2129
2130         /* Directly dispatch request retiring.  While we have the work queue
2131          * to handle this, the waiter on a request often wants an associated
2132          * buffer to have made it to the inactive list, and we would need
2133          * a separate wait queue to handle that.
2134          */
2135         if (ret == 0)
2136                 i915_gem_retire_requests_ring(dev, ring);
2137
2138         return ret;
2139 }
2140
2141 /**
2142  * Waits for a sequence number to be signaled, and cleans up the
2143  * request and object lists appropriately for that event.
2144  */
2145 static int
2146 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2147                   struct intel_ring_buffer *ring)
2148 {
2149         return i915_do_wait_request(dev, seqno, 1, ring);
2150 }
2151
2152 static void
2153 i915_gem_flush_ring(struct drm_device *dev,
2154                     struct drm_file *file_priv,
2155                     struct intel_ring_buffer *ring,
2156                     uint32_t invalidate_domains,
2157                     uint32_t flush_domains)
2158 {
2159         ring->flush(ring, invalidate_domains, flush_domains);
2160         i915_gem_process_flushing_list(dev, flush_domains, ring);
2161 }
2162
2163 static void
2164 i915_gem_flush(struct drm_device *dev,
2165                struct drm_file *file_priv,
2166                uint32_t invalidate_domains,
2167                uint32_t flush_domains,
2168                uint32_t flush_rings)
2169 {
2170         drm_i915_private_t *dev_priv = dev->dev_private;
2171
2172         if (flush_domains & I915_GEM_DOMAIN_CPU)
2173                 drm_agp_chipset_flush(dev);
2174
2175         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2176                 if (flush_rings & RING_RENDER)
2177                         i915_gem_flush_ring(dev, file_priv,
2178                                             &dev_priv->render_ring,
2179                                             invalidate_domains, flush_domains);
2180                 if (flush_rings & RING_BSD)
2181                         i915_gem_flush_ring(dev, file_priv,
2182                                             &dev_priv->bsd_ring,
2183                                             invalidate_domains, flush_domains);
2184                 if (flush_rings & RING_BLT)
2185                         i915_gem_flush_ring(dev, file_priv,
2186                                             &dev_priv->blt_ring,
2187                                             invalidate_domains, flush_domains);
2188         }
2189 }
2190
2191 /**
2192  * Ensures that all rendering to the object has completed and the object is
2193  * safe to unbind from the GTT or access from the CPU.
2194  */
2195 static int
2196 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2197                                bool interruptible)
2198 {
2199         struct drm_device *dev = obj->dev;
2200         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2201         int ret;
2202
2203         /* This function only exists to support waiting for existing rendering,
2204          * not for emitting required flushes.
2205          */
2206         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2207
2208         /* If there is rendering queued on the buffer being evicted, wait for
2209          * it.
2210          */
2211         if (obj_priv->active) {
2212                 ret = i915_do_wait_request(dev,
2213                                            obj_priv->last_rendering_seqno,
2214                                            interruptible,
2215                                            obj_priv->ring);
2216                 if (ret)
2217                         return ret;
2218         }
2219
2220         return 0;
2221 }
2222
2223 /**
2224  * Unbinds an object from the GTT aperture.
2225  */
2226 int
2227 i915_gem_object_unbind(struct drm_gem_object *obj)
2228 {
2229         struct drm_device *dev = obj->dev;
2230         struct drm_i915_private *dev_priv = dev->dev_private;
2231         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2232         int ret = 0;
2233
2234         if (obj_priv->gtt_space == NULL)
2235                 return 0;
2236
2237         if (obj_priv->pin_count != 0) {
2238                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2239                 return -EINVAL;
2240         }
2241
2242         /* blow away mappings if mapped through GTT */
2243         i915_gem_release_mmap(obj);
2244
2245         /* Move the object to the CPU domain to ensure that
2246          * any possible CPU writes while it's not in the GTT
2247          * are flushed when we go to remap it. This will
2248          * also ensure that all pending GPU writes are finished
2249          * before we unbind.
2250          */
2251         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2252         if (ret == -ERESTARTSYS)
2253                 return ret;
2254         /* Continue on if we fail due to EIO, the GPU is hung so we
2255          * should be safe and we need to cleanup or else we might
2256          * cause memory corruption through use-after-free.
2257          */
2258         if (ret) {
2259                 i915_gem_clflush_object(obj);
2260                 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2261         }
2262
2263         /* release the fence reg _after_ flushing */
2264         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2265                 i915_gem_clear_fence_reg(obj);
2266
2267         drm_unbind_agp(obj_priv->agp_mem);
2268         drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2269
2270         i915_gem_object_put_pages_gtt(obj);
2271
2272         i915_gem_info_remove_gtt(dev_priv, obj_priv);
2273         list_del_init(&obj_priv->mm_list);
2274         /* Avoid an unnecessary call to unbind on rebind. */
2275         obj_priv->map_and_fenceable = true;
2276
2277         drm_mm_put_block(obj_priv->gtt_space);
2278         obj_priv->gtt_space = NULL;
2279         obj_priv->gtt_offset = 0;
2280
2281         if (i915_gem_object_is_purgeable(obj_priv))
2282                 i915_gem_object_truncate(obj);
2283
2284         trace_i915_gem_object_unbind(obj);
2285
2286         return ret;
2287 }
2288
2289 static int i915_ring_idle(struct drm_device *dev,
2290                           struct intel_ring_buffer *ring)
2291 {
2292         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2293                 return 0;
2294
2295         i915_gem_flush_ring(dev, NULL, ring,
2296                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2297         return i915_wait_request(dev,
2298                                  i915_gem_next_request_seqno(dev, ring),
2299                                  ring);
2300 }
2301
2302 int
2303 i915_gpu_idle(struct drm_device *dev)
2304 {
2305         drm_i915_private_t *dev_priv = dev->dev_private;
2306         bool lists_empty;
2307         int ret;
2308
2309         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2310                        list_empty(&dev_priv->mm.active_list));
2311         if (lists_empty)
2312                 return 0;
2313
2314         /* Flush everything onto the inactive list. */
2315         ret = i915_ring_idle(dev, &dev_priv->render_ring);
2316         if (ret)
2317                 return ret;
2318
2319         ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2320         if (ret)
2321                 return ret;
2322
2323         ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2324         if (ret)
2325                 return ret;
2326
2327         return 0;
2328 }
2329
2330 static void sandybridge_write_fence_reg(struct drm_gem_object *obj)
2331 {
2332         struct drm_device *dev = obj->dev;
2333         drm_i915_private_t *dev_priv = dev->dev_private;
2334         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2335         u32 size = i915_gem_get_gtt_size(obj_priv);
2336         int regnum = obj_priv->fence_reg;
2337         uint64_t val;
2338
2339         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2340                     0xfffff000) << 32;
2341         val |= obj_priv->gtt_offset & 0xfffff000;
2342         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2343                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2344
2345         if (obj_priv->tiling_mode == I915_TILING_Y)
2346                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2347         val |= I965_FENCE_REG_VALID;
2348
2349         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2350 }
2351
2352 static void i965_write_fence_reg(struct drm_gem_object *obj)
2353 {
2354         struct drm_device *dev = obj->dev;
2355         drm_i915_private_t *dev_priv = dev->dev_private;
2356         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2357         u32 size = i915_gem_get_gtt_size(obj_priv);
2358         int regnum = obj_priv->fence_reg;
2359         uint64_t val;
2360
2361         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2362                     0xfffff000) << 32;
2363         val |= obj_priv->gtt_offset & 0xfffff000;
2364         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2365         if (obj_priv->tiling_mode == I915_TILING_Y)
2366                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2367         val |= I965_FENCE_REG_VALID;
2368
2369         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2370 }
2371
2372 static void i915_write_fence_reg(struct drm_gem_object *obj)
2373 {
2374         struct drm_device *dev = obj->dev;
2375         drm_i915_private_t *dev_priv = dev->dev_private;
2376         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2377         u32 size = i915_gem_get_gtt_size(obj_priv);
2378         uint32_t fence_reg, val, pitch_val;
2379         int tile_width;
2380
2381         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2382             (obj_priv->gtt_offset & (size - 1))) {
2383                 WARN(1, "%s: object 0x%08x [fenceable? %d] not 1M or size (0x%08x) aligned [gtt_space offset=%lx, size=%lx]\n",
2384                      __func__, obj_priv->gtt_offset, obj_priv->map_and_fenceable, size,
2385                      obj_priv->gtt_space->start, obj_priv->gtt_space->size);
2386                 return;
2387         }
2388
2389         if (obj_priv->tiling_mode == I915_TILING_Y &&
2390             HAS_128_BYTE_Y_TILING(dev))
2391                 tile_width = 128;
2392         else
2393                 tile_width = 512;
2394
2395         /* Note: pitch better be a power of two tile widths */
2396         pitch_val = obj_priv->stride / tile_width;
2397         pitch_val = ffs(pitch_val) - 1;
2398
2399         if (obj_priv->tiling_mode == I915_TILING_Y &&
2400             HAS_128_BYTE_Y_TILING(dev))
2401                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2402         else
2403                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2404
2405         val = obj_priv->gtt_offset;
2406         if (obj_priv->tiling_mode == I915_TILING_Y)
2407                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2408         val |= I915_FENCE_SIZE_BITS(size);
2409         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2410         val |= I830_FENCE_REG_VALID;
2411
2412         fence_reg = obj_priv->fence_reg;
2413         if (fence_reg < 8)
2414                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2415         else
2416                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2417         I915_WRITE(fence_reg, val);
2418 }
2419
2420 static void i830_write_fence_reg(struct drm_gem_object *obj)
2421 {
2422         struct drm_device *dev = obj->dev;
2423         drm_i915_private_t *dev_priv = dev->dev_private;
2424         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2425         u32 size = i915_gem_get_gtt_size(obj_priv);
2426         int regnum = obj_priv->fence_reg;
2427         uint32_t val;
2428         uint32_t pitch_val;
2429         uint32_t fence_size_bits;
2430
2431         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2432             (obj_priv->gtt_offset & (obj->size - 1))) {
2433                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2434                      __func__, obj_priv->gtt_offset);
2435                 return;
2436         }
2437
2438         pitch_val = obj_priv->stride / 128;
2439         pitch_val = ffs(pitch_val) - 1;
2440         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2441
2442         val = obj_priv->gtt_offset;
2443         if (obj_priv->tiling_mode == I915_TILING_Y)
2444                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2445         fence_size_bits = I830_FENCE_SIZE_BITS(size);
2446         WARN_ON(fence_size_bits & ~0x00000f00);
2447         val |= fence_size_bits;
2448         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2449         val |= I830_FENCE_REG_VALID;
2450
2451         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2452 }
2453
2454 static int i915_find_fence_reg(struct drm_device *dev,
2455                                bool interruptible)
2456 {
2457         struct drm_i915_private *dev_priv = dev->dev_private;
2458         struct drm_i915_fence_reg *reg;
2459         struct drm_i915_gem_object *obj_priv = NULL;
2460         int i, avail, ret;
2461
2462         /* First try to find a free reg */
2463         avail = 0;
2464         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2465                 reg = &dev_priv->fence_regs[i];
2466                 if (!reg->obj)
2467                         return i;
2468
2469                 obj_priv = to_intel_bo(reg->obj);
2470                 if (!obj_priv->pin_count)
2471                     avail++;
2472         }
2473
2474         if (avail == 0)
2475                 return -ENOSPC;
2476
2477         /* None available, try to steal one or wait for a user to finish */
2478         avail = I915_FENCE_REG_NONE;
2479         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2480                             lru_list) {
2481                 obj_priv = to_intel_bo(reg->obj);
2482                 if (obj_priv->pin_count)
2483                         continue;
2484
2485                 /* found one! */
2486                 avail = obj_priv->fence_reg;
2487                 break;
2488         }
2489
2490         BUG_ON(avail == I915_FENCE_REG_NONE);
2491
2492         /* We only have a reference on obj from the active list. put_fence_reg
2493          * might drop that one, causing a use-after-free in it. So hold a
2494          * private reference to obj like the other callers of put_fence_reg
2495          * (set_tiling ioctl) do. */
2496         drm_gem_object_reference(&obj_priv->base);
2497         ret = i915_gem_object_put_fence_reg(&obj_priv->base, interruptible);
2498         drm_gem_object_unreference(&obj_priv->base);
2499         if (ret != 0)
2500                 return ret;
2501
2502         return avail;
2503 }
2504
2505 /**
2506  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2507  * @obj: object to map through a fence reg
2508  *
2509  * When mapping objects through the GTT, userspace wants to be able to write
2510  * to them without having to worry about swizzling if the object is tiled.
2511  *
2512  * This function walks the fence regs looking for a free one for @obj,
2513  * stealing one if it can't find any.
2514  *
2515  * It then sets up the reg based on the object's properties: address, pitch
2516  * and tiling format.
2517  */
2518 int
2519 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2520                               bool interruptible)
2521 {
2522         struct drm_device *dev = obj->dev;
2523         struct drm_i915_private *dev_priv = dev->dev_private;
2524         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2525         struct drm_i915_fence_reg *reg = NULL;
2526         int ret;
2527
2528         /* Just update our place in the LRU if our fence is getting used. */
2529         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2530                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2531                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2532                 return 0;
2533         }
2534
2535         switch (obj_priv->tiling_mode) {
2536         case I915_TILING_NONE:
2537                 WARN(1, "allocating a fence for non-tiled object?\n");
2538                 break;
2539         case I915_TILING_X:
2540                 if (!obj_priv->stride)
2541                         return -EINVAL;
2542                 WARN((obj_priv->stride & (512 - 1)),
2543                      "object 0x%08x is X tiled but has non-512B pitch\n",
2544                      obj_priv->gtt_offset);
2545                 break;
2546         case I915_TILING_Y:
2547                 if (!obj_priv->stride)
2548                         return -EINVAL;
2549                 WARN((obj_priv->stride & (128 - 1)),
2550                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2551                      obj_priv->gtt_offset);
2552                 break;
2553         }
2554
2555         ret = i915_find_fence_reg(dev, interruptible);
2556         if (ret < 0)
2557                 return ret;
2558
2559         obj_priv->fence_reg = ret;
2560         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2561         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2562
2563         reg->obj = obj;
2564
2565         switch (INTEL_INFO(dev)->gen) {
2566         case 6:
2567                 sandybridge_write_fence_reg(obj);
2568                 break;
2569         case 5:
2570         case 4:
2571                 i965_write_fence_reg(obj);
2572                 break;
2573         case 3:
2574                 i915_write_fence_reg(obj);
2575                 break;
2576         case 2:
2577                 i830_write_fence_reg(obj);
2578                 break;
2579         }
2580
2581         trace_i915_gem_object_get_fence(obj,
2582                                         obj_priv->fence_reg,
2583                                         obj_priv->tiling_mode);
2584
2585         return 0;
2586 }
2587
2588 /**
2589  * i915_gem_clear_fence_reg - clear out fence register info
2590  * @obj: object to clear
2591  *
2592  * Zeroes out the fence register itself and clears out the associated
2593  * data structures in dev_priv and obj_priv.
2594  */
2595 static void
2596 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2597 {
2598         struct drm_device *dev = obj->dev;
2599         drm_i915_private_t *dev_priv = dev->dev_private;
2600         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2601         struct drm_i915_fence_reg *reg =
2602                 &dev_priv->fence_regs[obj_priv->fence_reg];
2603         uint32_t fence_reg;
2604
2605         switch (INTEL_INFO(dev)->gen) {
2606         case 6:
2607                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2608                              (obj_priv->fence_reg * 8), 0);
2609                 break;
2610         case 5:
2611         case 4:
2612                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2613                 break;
2614         case 3:
2615                 if (obj_priv->fence_reg >= 8)
2616                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2617                 else
2618         case 2:
2619                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2620
2621                 I915_WRITE(fence_reg, 0);
2622                 break;
2623         }
2624
2625         reg->obj = NULL;
2626         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2627         list_del_init(&reg->lru_list);
2628 }
2629
2630 /**
2631  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2632  * to the buffer to finish, and then resets the fence register.
2633  * @obj: tiled object holding a fence register.
2634  * @bool: whether the wait upon the fence is interruptible
2635  *
2636  * Zeroes out the fence register itself and clears out the associated
2637  * data structures in dev_priv and obj_priv.
2638  */
2639 int
2640 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2641                               bool interruptible)
2642 {
2643         struct drm_device *dev = obj->dev;
2644         struct drm_i915_private *dev_priv = dev->dev_private;
2645         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2646         struct drm_i915_fence_reg *reg;
2647
2648         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2649                 return 0;
2650
2651         /* If we've changed tiling, GTT-mappings of the object
2652          * need to re-fault to ensure that the correct fence register
2653          * setup is in place.
2654          */
2655         i915_gem_release_mmap(obj);
2656
2657         /* On the i915, GPU access to tiled buffers is via a fence,
2658          * therefore we must wait for any outstanding access to complete
2659          * before clearing the fence.
2660          */
2661         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2662         if (reg->gpu) {
2663                 int ret;
2664
2665                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2666                 if (ret)
2667                         return ret;
2668
2669                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2670                 if (ret)
2671                         return ret;
2672
2673                 reg->gpu = false;
2674         }
2675
2676         i915_gem_object_flush_gtt_write_domain(obj);
2677         i915_gem_clear_fence_reg(obj);
2678
2679         return 0;
2680 }
2681
2682 /**
2683  * Finds free space in the GTT aperture and binds the object there.
2684  */
2685 static int
2686 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
2687                             unsigned alignment,
2688                             bool map_and_fenceable)
2689 {
2690         struct drm_device *dev = obj->dev;
2691         drm_i915_private_t *dev_priv = dev->dev_private;
2692         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2693         struct drm_mm_node *free_space;
2694         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2695         u32 size, fence_size, fence_alignment;
2696         bool mappable, fenceable;
2697         int ret;
2698
2699         if (obj_priv->madv != I915_MADV_WILLNEED) {
2700                 DRM_ERROR("Attempting to bind a purgeable object\n");
2701                 return -EINVAL;
2702         }
2703
2704         fence_size = i915_gem_get_gtt_size(obj_priv);
2705         fence_alignment = i915_gem_get_gtt_alignment(obj_priv);
2706
2707         if (alignment == 0)
2708                 alignment = map_and_fenceable ? fence_alignment : 4096;
2709         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2710                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2711                 return -EINVAL;
2712         }
2713
2714         size = map_and_fenceable ? fence_size : obj->size;
2715
2716         /* If the object is bigger than the entire aperture, reject it early
2717          * before evicting everything in a vain attempt to find space.
2718          */
2719         if (obj->size >
2720             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2721                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2722                 return -E2BIG;
2723         }
2724
2725  search_free:
2726         if (map_and_fenceable)
2727                 free_space =
2728                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2729                                                     size, alignment, 0,
2730                                                     dev_priv->mm.gtt_mappable_end,
2731                                                     0);
2732         else
2733                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2734                                                 size, alignment, 0);
2735
2736         if (free_space != NULL) {
2737                 if (map_and_fenceable)
2738                         obj_priv->gtt_space =
2739                                 drm_mm_get_block_range_generic(free_space,
2740                                                                size, alignment, 0,
2741                                                                dev_priv->mm.gtt_mappable_end,
2742                                                                0);
2743                 else
2744                         obj_priv->gtt_space =
2745                                 drm_mm_get_block(free_space, size, alignment);
2746         }
2747         if (obj_priv->gtt_space == NULL) {
2748                 /* If the gtt is empty and we're still having trouble
2749                  * fitting our object in, we're out of memory.
2750                  */
2751                 ret = i915_gem_evict_something(dev, size, alignment,
2752                                                map_and_fenceable);
2753                 if (ret)
2754                         return ret;
2755
2756                 goto search_free;
2757         }
2758
2759         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2760         if (ret) {
2761                 drm_mm_put_block(obj_priv->gtt_space);
2762                 obj_priv->gtt_space = NULL;
2763
2764                 if (ret == -ENOMEM) {
2765                         /* first try to clear up some space from the GTT */
2766                         ret = i915_gem_evict_something(dev, size,
2767                                                        alignment,
2768                                                        map_and_fenceable);
2769                         if (ret) {
2770                                 /* now try to shrink everyone else */
2771                                 if (gfpmask) {
2772                                         gfpmask = 0;
2773                                         goto search_free;
2774                                 }
2775
2776                                 return ret;
2777                         }
2778
2779                         goto search_free;
2780                 }
2781
2782                 return ret;
2783         }
2784
2785         /* Create an AGP memory structure pointing at our pages, and bind it
2786          * into the GTT.
2787          */
2788         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2789                                                obj_priv->pages,
2790                                                obj->size >> PAGE_SHIFT,
2791                                                obj_priv->gtt_space->start,
2792                                                obj_priv->agp_type);
2793         if (obj_priv->agp_mem == NULL) {
2794                 i915_gem_object_put_pages_gtt(obj);
2795                 drm_mm_put_block(obj_priv->gtt_space);
2796                 obj_priv->gtt_space = NULL;
2797
2798                 ret = i915_gem_evict_something(dev, size,
2799                                                alignment, map_and_fenceable);
2800                 if (ret)
2801                         return ret;
2802
2803                 goto search_free;
2804         }
2805
2806         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2807
2808         /* keep track of bounds object by adding it to the inactive list */
2809         list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
2810         i915_gem_info_add_gtt(dev_priv, obj_priv);
2811
2812         /* Assert that the object is not currently in any GPU domain. As it
2813          * wasn't in the GTT, there shouldn't be any way it could have been in
2814          * a GPU cache
2815          */
2816         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2817         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2818
2819         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset, map_and_fenceable);
2820
2821         fenceable =
2822                 obj_priv->gtt_space->size == fence_size &&
2823                 (obj_priv->gtt_space->start & (fence_alignment -1)) == 0;
2824
2825         mappable =
2826                 obj_priv->gtt_offset + obj->size <= dev_priv->mm.gtt_mappable_end;
2827
2828         obj_priv->map_and_fenceable = mappable && fenceable;
2829
2830         return 0;
2831 }
2832
2833 void
2834 i915_gem_clflush_object(struct drm_gem_object *obj)
2835 {
2836         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2837
2838         /* If we don't have a page list set up, then we're not pinned
2839          * to GPU, and we can ignore the cache flush because it'll happen
2840          * again at bind time.
2841          */
2842         if (obj_priv->pages == NULL)
2843                 return;
2844
2845         trace_i915_gem_object_clflush(obj);
2846
2847         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2848 }
2849
2850 /** Flushes any GPU write domain for the object if it's dirty. */
2851 static int
2852 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2853                                        bool pipelined)
2854 {
2855         struct drm_device *dev = obj->dev;
2856
2857         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2858                 return 0;
2859
2860         /* Queue the GPU write cache flushing we need. */
2861         i915_gem_flush_ring(dev, NULL,
2862                             to_intel_bo(obj)->ring,
2863                             0, obj->write_domain);
2864         BUG_ON(obj->write_domain);
2865
2866         if (pipelined)
2867                 return 0;
2868
2869         return i915_gem_object_wait_rendering(obj, true);
2870 }
2871
2872 /** Flushes the GTT write domain for the object if it's dirty. */
2873 static void
2874 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2875 {
2876         uint32_t old_write_domain;
2877
2878         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2879                 return;
2880
2881         /* No actual flushing is required for the GTT write domain.   Writes
2882          * to it immediately go to main memory as far as we know, so there's
2883          * no chipset flush.  It also doesn't land in render cache.
2884          */
2885         i915_gem_release_mmap(obj);
2886
2887         old_write_domain = obj->write_domain;
2888         obj->write_domain = 0;
2889
2890         trace_i915_gem_object_change_domain(obj,
2891                                             obj->read_domains,
2892                                             old_write_domain);
2893 }
2894
2895 /** Flushes the CPU write domain for the object if it's dirty. */
2896 static void
2897 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2898 {
2899         struct drm_device *dev = obj->dev;
2900         uint32_t old_write_domain;
2901
2902         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2903                 return;
2904
2905         i915_gem_clflush_object(obj);
2906         drm_agp_chipset_flush(dev);
2907         old_write_domain = obj->write_domain;
2908         obj->write_domain = 0;
2909
2910         trace_i915_gem_object_change_domain(obj,
2911                                             obj->read_domains,
2912                                             old_write_domain);
2913 }
2914
2915 /**
2916  * Moves a single object to the GTT read, and possibly write domain.
2917  *
2918  * This function returns when the move is complete, including waiting on
2919  * flushes to occur.
2920  */
2921 int
2922 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2923 {
2924         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2925         uint32_t old_write_domain, old_read_domains;
2926         int ret;
2927
2928         /* Not valid to be called on unbound objects. */
2929         if (obj_priv->gtt_space == NULL)
2930                 return -EINVAL;
2931
2932         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2933         if (ret != 0)
2934                 return ret;
2935
2936         i915_gem_object_flush_cpu_write_domain(obj);
2937
2938         if (write) {
2939                 ret = i915_gem_object_wait_rendering(obj, true);
2940                 if (ret)
2941                         return ret;
2942         }
2943
2944         old_write_domain = obj->write_domain;
2945         old_read_domains = obj->read_domains;
2946
2947         /* It should now be out of any other write domains, and we can update
2948          * the domain values for our changes.
2949          */
2950         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2951         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2952         if (write) {
2953                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2954                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2955                 obj_priv->dirty = 1;
2956         }
2957
2958         trace_i915_gem_object_change_domain(obj,
2959                                             old_read_domains,
2960                                             old_write_domain);
2961
2962         return 0;
2963 }
2964
2965 /*
2966  * Prepare buffer for display plane. Use uninterruptible for possible flush
2967  * wait, as in modesetting process we're not supposed to be interrupted.
2968  */
2969 int
2970 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2971                                      bool pipelined)
2972 {
2973         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2974         uint32_t old_read_domains;
2975         int ret;
2976
2977         /* Not valid to be called on unbound objects. */
2978         if (obj_priv->gtt_space == NULL)
2979                 return -EINVAL;
2980
2981         ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2982         if (ret)
2983                 return ret;
2984
2985         /* Currently, we are always called from an non-interruptible context. */
2986         if (!pipelined) {
2987                 ret = i915_gem_object_wait_rendering(obj, false);
2988                 if (ret)
2989                         return ret;
2990         }
2991
2992         i915_gem_object_flush_cpu_write_domain(obj);
2993
2994         old_read_domains = obj->read_domains;
2995         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2996
2997         trace_i915_gem_object_change_domain(obj,
2998                                             old_read_domains,
2999                                             obj->write_domain);
3000
3001         return 0;
3002 }
3003
3004 /**
3005  * Moves a single object to the CPU read, and possibly write domain.
3006  *
3007  * This function returns when the move is complete, including waiting on
3008  * flushes to occur.
3009  */
3010 static int
3011 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
3012 {
3013         uint32_t old_write_domain, old_read_domains;
3014         int ret;
3015
3016         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3017         if (ret != 0)
3018                 return ret;
3019
3020         i915_gem_object_flush_gtt_write_domain(obj);
3021
3022         /* If we have a partially-valid cache of the object in the CPU,
3023          * finish invalidating it and free the per-page flags.
3024          */
3025         i915_gem_object_set_to_full_cpu_read_domain(obj);
3026
3027         if (write) {
3028                 ret = i915_gem_object_wait_rendering(obj, true);
3029                 if (ret)
3030                         return ret;
3031         }
3032
3033         old_write_domain = obj->write_domain;
3034         old_read_domains = obj->read_domains;
3035
3036         /* Flush the CPU cache if it's still invalid. */
3037         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3038                 i915_gem_clflush_object(obj);
3039
3040                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3041         }
3042
3043         /* It should now be out of any other write domains, and we can update
3044          * the domain values for our changes.
3045          */
3046         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3047
3048         /* If we're writing through the CPU, then the GPU read domains will
3049          * need to be invalidated at next use.
3050          */
3051         if (write) {
3052                 obj->read_domains = I915_GEM_DOMAIN_CPU;
3053                 obj->write_domain = I915_GEM_DOMAIN_CPU;
3054         }
3055
3056         trace_i915_gem_object_change_domain(obj,
3057                                             old_read_domains,
3058                                             old_write_domain);
3059
3060         return 0;
3061 }
3062
3063 /*
3064  * Set the next domain for the specified object. This
3065  * may not actually perform the necessary flushing/invaliding though,
3066  * as that may want to be batched with other set_domain operations
3067  *
3068  * This is (we hope) the only really tricky part of gem. The goal
3069  * is fairly simple -- track which caches hold bits of the object
3070  * and make sure they remain coherent. A few concrete examples may
3071  * help to explain how it works. For shorthand, we use the notation
3072  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3073  * a pair of read and write domain masks.
3074  *
3075  * Case 1: the batch buffer
3076  *
3077  *      1. Allocated
3078  *      2. Written by CPU
3079  *      3. Mapped to GTT
3080  *      4. Read by GPU
3081  *      5. Unmapped from GTT
3082  *      6. Freed
3083  *
3084  *      Let's take these a step at a time
3085  *
3086  *      1. Allocated
3087  *              Pages allocated from the kernel may still have
3088  *              cache contents, so we set them to (CPU, CPU) always.
3089  *      2. Written by CPU (using pwrite)
3090  *              The pwrite function calls set_domain (CPU, CPU) and
3091  *              this function does nothing (as nothing changes)
3092  *      3. Mapped by GTT
3093  *              This function asserts that the object is not
3094  *              currently in any GPU-based read or write domains
3095  *      4. Read by GPU
3096  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3097  *              As write_domain is zero, this function adds in the
3098  *              current read domains (CPU+COMMAND, 0).
3099  *              flush_domains is set to CPU.
3100  *              invalidate_domains is set to COMMAND
3101  *              clflush is run to get data out of the CPU caches
3102  *              then i915_dev_set_domain calls i915_gem_flush to
3103  *              emit an MI_FLUSH and drm_agp_chipset_flush
3104  *      5. Unmapped from GTT
3105  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3106  *              flush_domains and invalidate_domains end up both zero
3107  *              so no flushing/invalidating happens
3108  *      6. Freed
3109  *              yay, done
3110  *
3111  * Case 2: The shared render buffer
3112  *
3113  *      1. Allocated
3114  *      2. Mapped to GTT
3115  *      3. Read/written by GPU
3116  *      4. set_domain to (CPU,CPU)
3117  *      5. Read/written by CPU
3118  *      6. Read/written by GPU
3119  *
3120  *      1. Allocated
3121  *              Same as last example, (CPU, CPU)
3122  *      2. Mapped to GTT
3123  *              Nothing changes (assertions find that it is not in the GPU)
3124  *      3. Read/written by GPU
3125  *              execbuffer calls set_domain (RENDER, RENDER)
3126  *              flush_domains gets CPU
3127  *              invalidate_domains gets GPU
3128  *              clflush (obj)
3129  *              MI_FLUSH and drm_agp_chipset_flush
3130  *      4. set_domain (CPU, CPU)
3131  *              flush_domains gets GPU
3132  *              invalidate_domains gets CPU
3133  *              wait_rendering (obj) to make sure all drawing is complete.
3134  *              This will include an MI_FLUSH to get the data from GPU
3135  *              to memory
3136  *              clflush (obj) to invalidate the CPU cache
3137  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3138  *      5. Read/written by CPU
3139  *              cache lines are loaded and dirtied
3140  *      6. Read written by GPU
3141  *              Same as last GPU access
3142  *
3143  * Case 3: The constant buffer
3144  *
3145  *      1. Allocated
3146  *      2. Written by CPU
3147  *      3. Read by GPU
3148  *      4. Updated (written) by CPU again
3149  *      5. Read by GPU
3150  *
3151  *      1. Allocated
3152  *              (CPU, CPU)
3153  *      2. Written by CPU
3154  *              (CPU, CPU)
3155  *      3. Read by GPU
3156  *              (CPU+RENDER, 0)
3157  *              flush_domains = CPU
3158  *              invalidate_domains = RENDER
3159  *              clflush (obj)
3160  *              MI_FLUSH
3161  *              drm_agp_chipset_flush
3162  *      4. Updated (written) by CPU again
3163  *              (CPU, CPU)
3164  *              flush_domains = 0 (no previous write domain)
3165  *              invalidate_domains = 0 (no new read domains)
3166  *      5. Read by GPU
3167  *              (CPU+RENDER, 0)
3168  *              flush_domains = CPU
3169  *              invalidate_domains = RENDER
3170  *              clflush (obj)
3171  *              MI_FLUSH
3172  *              drm_agp_chipset_flush
3173  */
3174 static void
3175 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3176                                   struct intel_ring_buffer *ring,
3177                                   struct change_domains *cd)
3178 {
3179         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3180         uint32_t                        invalidate_domains = 0;
3181         uint32_t                        flush_domains = 0;
3182
3183         /*
3184          * If the object isn't moving to a new write domain,
3185          * let the object stay in multiple read domains
3186          */
3187         if (obj->pending_write_domain == 0)
3188                 obj->pending_read_domains |= obj->read_domains;
3189
3190         /*
3191          * Flush the current write domain if
3192          * the new read domains don't match. Invalidate
3193          * any read domains which differ from the old
3194          * write domain
3195          */
3196         if (obj->write_domain &&
3197             (obj->write_domain != obj->pending_read_domains ||
3198              obj_priv->ring != ring)) {
3199                 flush_domains |= obj->write_domain;
3200                 invalidate_domains |=
3201                         obj->pending_read_domains & ~obj->write_domain;
3202         }
3203         /*
3204          * Invalidate any read caches which may have
3205          * stale data. That is, any new read domains.
3206          */
3207         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3208         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3209                 i915_gem_clflush_object(obj);
3210
3211         /* blow away mappings if mapped through GTT */
3212         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
3213                 i915_gem_release_mmap(obj);
3214
3215         /* The actual obj->write_domain will be updated with
3216          * pending_write_domain after we emit the accumulated flush for all
3217          * of our domain changes in execbuffers (which clears objects'
3218          * write_domains).  So if we have a current write domain that we
3219          * aren't changing, set pending_write_domain to that.
3220          */
3221         if (flush_domains == 0 && obj->pending_write_domain == 0)
3222                 obj->pending_write_domain = obj->write_domain;
3223
3224         cd->invalidate_domains |= invalidate_domains;
3225         cd->flush_domains |= flush_domains;
3226         if (flush_domains & I915_GEM_GPU_DOMAINS)
3227                 cd->flush_rings |= obj_priv->ring->id;
3228         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3229                 cd->flush_rings |= ring->id;
3230 }
3231
3232 /**
3233  * Moves the object from a partially CPU read to a full one.
3234  *
3235  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3236  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3237  */
3238 static void
3239 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3240 {
3241         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3242
3243         if (!obj_priv->page_cpu_valid)
3244                 return;
3245
3246         /* If we're partially in the CPU read domain, finish moving it in.
3247          */
3248         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3249                 int i;
3250
3251                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3252                         if (obj_priv->page_cpu_valid[i])
3253                                 continue;
3254                         drm_clflush_pages(obj_priv->pages + i, 1);
3255                 }
3256         }
3257
3258         /* Free the page_cpu_valid mappings which are now stale, whether
3259          * or not we've got I915_GEM_DOMAIN_CPU.
3260          */
3261         kfree(obj_priv->page_cpu_valid);
3262         obj_priv->page_cpu_valid = NULL;
3263 }
3264
3265 /**
3266  * Set the CPU read domain on a range of the object.
3267  *
3268  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3269  * not entirely valid.  The page_cpu_valid member of the object flags which
3270  * pages have been flushed, and will be respected by
3271  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3272  * of the whole object.
3273  *
3274  * This function returns when the move is complete, including waiting on
3275  * flushes to occur.
3276  */
3277 static int
3278 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3279                                           uint64_t offset, uint64_t size)
3280 {
3281         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3282         uint32_t old_read_domains;
3283         int i, ret;
3284
3285         if (offset == 0 && size == obj->size)
3286                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3287
3288         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3289         if (ret != 0)
3290                 return ret;
3291         i915_gem_object_flush_gtt_write_domain(obj);
3292
3293         /* If we're already fully in the CPU read domain, we're done. */
3294         if (obj_priv->page_cpu_valid == NULL &&
3295             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3296                 return 0;
3297
3298         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3299          * newly adding I915_GEM_DOMAIN_CPU
3300          */
3301         if (obj_priv->page_cpu_valid == NULL) {
3302                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3303                                                    GFP_KERNEL);
3304                 if (obj_priv->page_cpu_valid == NULL)
3305                         return -ENOMEM;
3306         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3307                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3308
3309         /* Flush the cache on any pages that are still invalid from the CPU's
3310          * perspective.
3311          */
3312         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3313              i++) {
3314                 if (obj_priv->page_cpu_valid[i])
3315                         continue;
3316
3317                 drm_clflush_pages(obj_priv->pages + i, 1);
3318
3319                 obj_priv->page_cpu_valid[i] = 1;
3320         }
3321
3322         /* It should now be out of any other write domains, and we can update
3323          * the domain values for our changes.
3324          */
3325         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3326
3327         old_read_domains = obj->read_domains;
3328         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3329
3330         trace_i915_gem_object_change_domain(obj,
3331                                             old_read_domains,
3332                                             obj->write_domain);
3333
3334         return 0;
3335 }
3336
3337 /**
3338  * Pin an object to the GTT and evaluate the relocations landing in it.
3339  */
3340 static int
3341 i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
3342                              struct drm_file *file_priv,
3343                              struct drm_i915_gem_exec_object2 *entry)
3344 {
3345         struct drm_device *dev = obj->base.dev;
3346         drm_i915_private_t *dev_priv = dev->dev_private;
3347         struct drm_i915_gem_relocation_entry __user *user_relocs;
3348         struct drm_gem_object *target_obj = NULL;
3349         uint32_t target_handle = 0;
3350         int i, ret = 0;
3351
3352         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3353         for (i = 0; i < entry->relocation_count; i++) {
3354                 struct drm_i915_gem_relocation_entry reloc;
3355                 uint32_t target_offset;
3356
3357                 if (__copy_from_user_inatomic(&reloc,
3358                                               user_relocs+i,
3359                                               sizeof(reloc))) {
3360                         ret = -EFAULT;
3361                         break;
3362                 }
3363
3364                 if (reloc.target_handle != target_handle) {
3365                         drm_gem_object_unreference(target_obj);
3366
3367                         target_obj = drm_gem_object_lookup(dev, file_priv,
3368                                                            reloc.target_handle);
3369                         if (target_obj == NULL) {
3370                                 ret = -ENOENT;
3371                                 break;
3372                         }
3373
3374                         target_handle = reloc.target_handle;
3375                 }
3376                 target_offset = to_intel_bo(target_obj)->gtt_offset;
3377
3378 #if WATCH_RELOC
3379                 DRM_INFO("%s: obj %p offset %08x target %d "
3380                          "read %08x write %08x gtt %08x "
3381                          "presumed %08x delta %08x\n",
3382                          __func__,
3383                          obj,
3384                          (int) reloc.offset,
3385                          (int) reloc.target_handle,
3386                          (int) reloc.read_domains,
3387                          (int) reloc.write_domain,
3388                          (int) target_offset,
3389                          (int) reloc.presumed_offset,
3390                          reloc.delta);
3391 #endif
3392
3393                 /* The target buffer should have appeared before us in the
3394                  * exec_object list, so it should have a GTT space bound by now.
3395                  */
3396                 if (target_offset == 0) {
3397                         DRM_ERROR("No GTT space found for object %d\n",
3398                                   reloc.target_handle);
3399                         ret = -EINVAL;
3400                         break;
3401                 }
3402
3403                 /* Validate that the target is in a valid r/w GPU domain */
3404                 if (reloc.write_domain & (reloc.write_domain - 1)) {
3405                         DRM_ERROR("reloc with multiple write domains: "
3406                                   "obj %p target %d offset %d "
3407                                   "read %08x write %08x",
3408                                   obj, reloc.target_handle,
3409                                   (int) reloc.offset,
3410                                   reloc.read_domains,
3411                                   reloc.write_domain);
3412                         ret = -EINVAL;
3413                         break;
3414                 }
3415                 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3416                     reloc.read_domains & I915_GEM_DOMAIN_CPU) {
3417                         DRM_ERROR("reloc with read/write CPU domains: "
3418                                   "obj %p target %d offset %d "
3419                                   "read %08x write %08x",
3420                                   obj, reloc.target_handle,
3421                                   (int) reloc.offset,
3422                                   reloc.read_domains,
3423                                   reloc.write_domain);
3424                         ret = -EINVAL;
3425                         break;
3426                 }
3427                 if (reloc.write_domain && target_obj->pending_write_domain &&
3428                     reloc.write_domain != target_obj->pending_write_domain) {
3429                         DRM_ERROR("Write domain conflict: "
3430                                   "obj %p target %d offset %d "
3431                                   "new %08x old %08x\n",
3432                                   obj, reloc.target_handle,
3433                                   (int) reloc.offset,
3434                                   reloc.write_domain,
3435                                   target_obj->pending_write_domain);
3436                         ret = -EINVAL;
3437                         break;
3438                 }
3439
3440                 target_obj->pending_read_domains |= reloc.read_domains;
3441                 target_obj->pending_write_domain |= reloc.write_domain;
3442
3443                 /* If the relocation already has the right value in it, no
3444                  * more work needs to be done.
3445                  */
3446                 if (target_offset == reloc.presumed_offset)
3447                         continue;
3448
3449                 /* Check that the relocation address is valid... */
3450                 if (reloc.offset > obj->base.size - 4) {
3451                         DRM_ERROR("Relocation beyond object bounds: "
3452                                   "obj %p target %d offset %d size %d.\n",
3453                                   obj, reloc.target_handle,
3454                                   (int) reloc.offset, (int) obj->base.size);
3455                         ret = -EINVAL;
3456                         break;
3457                 }
3458                 if (reloc.offset & 3) {
3459                         DRM_ERROR("Relocation not 4-byte aligned: "
3460                                   "obj %p target %d offset %d.\n",
3461                                   obj, reloc.target_handle,
3462                                   (int) reloc.offset);
3463                         ret = -EINVAL;
3464                         break;
3465                 }
3466
3467                 /* and points to somewhere within the target object. */
3468                 if (reloc.delta >= target_obj->size) {
3469                         DRM_ERROR("Relocation beyond target object bounds: "
3470                                   "obj %p target %d delta %d size %d.\n",
3471                                   obj, reloc.target_handle,
3472                                   (int) reloc.delta, (int) target_obj->size);
3473                         ret = -EINVAL;
3474                         break;
3475                 }
3476
3477                 reloc.delta += target_offset;
3478                 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3479                         uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3480                         char *vaddr;
3481
3482                         vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT]);
3483                         *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3484                         kunmap_atomic(vaddr);
3485                 } else {
3486                         uint32_t __iomem *reloc_entry;
3487                         void __iomem *reloc_page;
3488
3489                         ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3490                         if (ret)
3491                                 break;
3492
3493                         /* Map the page containing the relocation we're going to perform.  */
3494                         reloc.offset += obj->gtt_offset;
3495                         reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3496                                                               reloc.offset & PAGE_MASK);
3497                         reloc_entry = (uint32_t __iomem *)
3498                                 (reloc_page + (reloc.offset & ~PAGE_MASK));
3499                         iowrite32(reloc.delta, reloc_entry);
3500                         io_mapping_unmap_atomic(reloc_page);
3501                 }
3502
3503                 /* and update the user's relocation entry */
3504                 reloc.presumed_offset = target_offset;
3505                 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3506                                               &reloc.presumed_offset,
3507                                               sizeof(reloc.presumed_offset))) {
3508                     ret = -EFAULT;
3509                     break;
3510                 }
3511         }
3512
3513         drm_gem_object_unreference(target_obj);
3514         return ret;
3515 }
3516
3517 static int
3518 i915_gem_execbuffer_pin(struct drm_device *dev,
3519                         struct drm_file *file,
3520                         struct drm_gem_object **object_list,
3521                         struct drm_i915_gem_exec_object2 *exec_list,
3522                         int count)
3523 {
3524         struct drm_i915_private *dev_priv = dev->dev_private;
3525         int ret, i, retry;
3526
3527         /* attempt to pin all of the buffers into the GTT */
3528         retry = 0;
3529         do {
3530                 ret = 0;
3531                 for (i = 0; i < count; i++) {
3532                         struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3533                         struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3534                         bool need_fence =
3535                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3536                                 obj->tiling_mode != I915_TILING_NONE;
3537
3538                         /* g33/pnv can't fence buffers in the unmappable part */
3539                         bool need_mappable =
3540                                 entry->relocation_count ? true : need_fence;
3541
3542                         /* Check fence reg constraints and rebind if necessary */
3543                         if (need_mappable && !obj->map_and_fenceable) {
3544                                 ret = i915_gem_object_unbind(&obj->base);
3545                                 if (ret)
3546                                         break;
3547                         }
3548
3549                         ret = i915_gem_object_pin(&obj->base,
3550                                                   entry->alignment,
3551                                                   need_mappable);
3552                         if (ret)
3553                                 break;
3554
3555                         /*
3556                          * Pre-965 chips need a fence register set up in order
3557                          * to properly handle blits to/from tiled surfaces.
3558                          */
3559                         if (need_fence) {
3560                                 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3561                                 if (ret) {
3562                                         i915_gem_object_unpin(&obj->base);
3563                                         break;
3564                                 }
3565
3566                                 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3567                         }
3568
3569                         entry->offset = obj->gtt_offset;
3570                 }
3571
3572                 while (i--)
3573                         i915_gem_object_unpin(object_list[i]);
3574
3575                 if (ret != -ENOSPC || retry > 1)
3576                         return ret;
3577
3578                 /* First attempt, just clear anything that is purgeable.
3579                  * Second attempt, clear the entire GTT.
3580                  */
3581                 ret = i915_gem_evict_everything(dev, retry == 0);
3582                 if (ret)
3583                         return ret;
3584
3585                 retry++;
3586         } while (1);
3587 }
3588
3589 static int
3590 i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3591                                 struct drm_file *file,
3592                                 struct intel_ring_buffer *ring,
3593                                 struct drm_gem_object **objects,
3594                                 int count)
3595 {
3596         struct change_domains cd;
3597         int ret, i;
3598
3599         cd.invalidate_domains = 0;
3600         cd.flush_domains = 0;
3601         cd.flush_rings = 0;
3602         for (i = 0; i < count; i++)
3603                 i915_gem_object_set_to_gpu_domain(objects[i], ring, &cd);
3604
3605         if (cd.invalidate_domains | cd.flush_domains) {
3606 #if WATCH_EXEC
3607                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3608                           __func__,
3609                          cd.invalidate_domains,
3610                          cd.flush_domains);
3611 #endif
3612                 i915_gem_flush(dev, file,
3613                                cd.invalidate_domains,
3614                                cd.flush_domains,
3615                                cd.flush_rings);
3616         }
3617
3618         for (i = 0; i < count; i++) {
3619                 struct drm_i915_gem_object *obj = to_intel_bo(objects[i]);
3620                 /* XXX replace with semaphores */
3621                 if (obj->ring && ring != obj->ring) {
3622                         ret = i915_gem_object_wait_rendering(&obj->base, true);
3623                         if (ret)
3624                                 return ret;
3625                 }
3626         }
3627
3628         return 0;
3629 }
3630
3631 /* Throttle our rendering by waiting until the ring has completed our requests
3632  * emitted over 20 msec ago.
3633  *
3634  * Note that if we were to use the current jiffies each time around the loop,
3635  * we wouldn't escape the function with any frames outstanding if the time to
3636  * render a frame was over 20ms.
3637  *
3638  * This should get us reasonable parallelism between CPU and GPU but also
3639  * relatively low latency when blocking on a particular request to finish.
3640  */
3641 static int
3642 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3643 {
3644         struct drm_i915_private *dev_priv = dev->dev_private;
3645         struct drm_i915_file_private *file_priv = file->driver_priv;
3646         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3647         struct drm_i915_gem_request *request;
3648         struct intel_ring_buffer *ring = NULL;
3649         u32 seqno = 0;
3650         int ret;
3651
3652         spin_lock(&file_priv->mm.lock);
3653         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3654                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3655                         break;
3656
3657                 ring = request->ring;
3658                 seqno = request->seqno;
3659         }
3660         spin_unlock(&file_priv->mm.lock);
3661
3662         if (seqno == 0)
3663                 return 0;
3664
3665         ret = 0;
3666         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3667                 /* And wait for the seqno passing without holding any locks and
3668                  * causing extra latency for others. This is safe as the irq
3669                  * generation is designed to be run atomically and so is
3670                  * lockless.
3671                  */
3672                 ring->user_irq_get(ring);
3673                 ret = wait_event_interruptible(ring->irq_queue,
3674                                                i915_seqno_passed(ring->get_seqno(ring), seqno)
3675                                                || atomic_read(&dev_priv->mm.wedged));
3676                 ring->user_irq_put(ring);
3677
3678                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3679                         ret = -EIO;
3680         }
3681
3682         if (ret == 0)
3683                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3684
3685         return ret;
3686 }
3687
3688 static int
3689 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3690                           uint64_t exec_offset)
3691 {
3692         uint32_t exec_start, exec_len;
3693
3694         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3695         exec_len = (uint32_t) exec->batch_len;
3696
3697         if ((exec_start | exec_len) & 0x7)
3698                 return -EINVAL;
3699
3700         if (!exec_start)
3701                 return -EINVAL;
3702
3703         return 0;
3704 }
3705
3706 static int
3707 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3708                    int count)
3709 {
3710         int i;
3711
3712         for (i = 0; i < count; i++) {
3713                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3714                 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
3715
3716                 if (!access_ok(VERIFY_READ, ptr, length))
3717                         return -EFAULT;
3718
3719                 /* we may also need to update the presumed offsets */
3720                 if (!access_ok(VERIFY_WRITE, ptr, length))
3721                         return -EFAULT;
3722
3723                 if (fault_in_pages_readable(ptr, length))
3724                         return -EFAULT;
3725         }
3726
3727         return 0;
3728 }
3729
3730 static int
3731 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3732                        struct drm_file *file,
3733                        struct drm_i915_gem_execbuffer2 *args,
3734                        struct drm_i915_gem_exec_object2 *exec_list)
3735 {
3736         drm_i915_private_t *dev_priv = dev->dev_private;
3737         struct drm_gem_object **object_list = NULL;
3738         struct drm_gem_object *batch_obj;
3739         struct drm_clip_rect *cliprects = NULL;
3740         struct drm_i915_gem_request *request = NULL;
3741         int ret, i, flips;
3742         uint64_t exec_offset;
3743
3744         struct intel_ring_buffer *ring = NULL;
3745
3746         ret = i915_gem_check_is_wedged(dev);
3747         if (ret)
3748                 return ret;
3749
3750         ret = validate_exec_list(exec_list, args->buffer_count);
3751         if (ret)
3752                 return ret;
3753
3754 #if WATCH_EXEC
3755         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3756                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3757 #endif
3758         switch (args->flags & I915_EXEC_RING_MASK) {
3759         case I915_EXEC_DEFAULT:
3760         case I915_EXEC_RENDER:
3761                 ring = &dev_priv->render_ring;
3762                 break;
3763         case I915_EXEC_BSD:
3764                 if (!HAS_BSD(dev)) {
3765                         DRM_ERROR("execbuf with invalid ring (BSD)\n");
3766                         return -EINVAL;
3767                 }
3768                 ring = &dev_priv->bsd_ring;
3769                 break;
3770         case I915_EXEC_BLT:
3771                 if (!HAS_BLT(dev)) {
3772                         DRM_ERROR("execbuf with invalid ring (BLT)\n");
3773                         return -EINVAL;
3774                 }
3775                 ring = &dev_priv->blt_ring;
3776                 break;
3777         default:
3778                 DRM_ERROR("execbuf with unknown ring: %d\n",
3779                           (int)(args->flags & I915_EXEC_RING_MASK));
3780                 return -EINVAL;
3781         }
3782
3783         if (args->buffer_count < 1) {
3784                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3785                 return -EINVAL;
3786         }
3787         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3788         if (object_list == NULL) {
3789                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3790                           args->buffer_count);
3791                 ret = -ENOMEM;
3792                 goto pre_mutex_err;
3793         }
3794
3795         if (args->num_cliprects != 0) {
3796                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3797                                     GFP_KERNEL);
3798                 if (cliprects == NULL) {
3799                         ret = -ENOMEM;
3800                         goto pre_mutex_err;
3801                 }
3802
3803                 ret = copy_from_user(cliprects,
3804                                      (struct drm_clip_rect __user *)
3805                                      (uintptr_t) args->cliprects_ptr,
3806                                      sizeof(*cliprects) * args->num_cliprects);
3807                 if (ret != 0) {
3808                         DRM_ERROR("copy %d cliprects failed: %d\n",
3809                                   args->num_cliprects, ret);
3810                         ret = -EFAULT;
3811                         goto pre_mutex_err;
3812                 }
3813         }
3814
3815         request = kzalloc(sizeof(*request), GFP_KERNEL);
3816         if (request == NULL) {
3817                 ret = -ENOMEM;
3818                 goto pre_mutex_err;
3819         }
3820
3821         ret = i915_mutex_lock_interruptible(dev);
3822         if (ret)
3823                 goto pre_mutex_err;
3824
3825         if (dev_priv->mm.suspended) {
3826                 mutex_unlock(&dev->struct_mutex);
3827                 ret = -EBUSY;
3828                 goto pre_mutex_err;
3829         }
3830
3831         /* Look up object handles */
3832         for (i = 0; i < args->buffer_count; i++) {
3833                 struct drm_i915_gem_object *obj_priv;
3834
3835                 object_list[i] = drm_gem_object_lookup(dev, file,
3836                                                        exec_list[i].handle);
3837                 if (object_list[i] == NULL) {
3838                         DRM_ERROR("Invalid object handle %d at index %d\n",
3839                                    exec_list[i].handle, i);
3840                         /* prevent error path from reading uninitialized data */
3841                         args->buffer_count = i + 1;
3842                         ret = -ENOENT;
3843                         goto err;
3844                 }
3845
3846                 obj_priv = to_intel_bo(object_list[i]);
3847                 if (obj_priv->in_execbuffer) {
3848                         DRM_ERROR("Object %p appears more than once in object list\n",
3849                                    object_list[i]);
3850                         /* prevent error path from reading uninitialized data */
3851                         args->buffer_count = i + 1;
3852                         ret = -EINVAL;
3853                         goto err;
3854                 }
3855                 obj_priv->in_execbuffer = true;
3856         }
3857
3858         /* Move the objects en-masse into the GTT, evicting if necessary. */
3859         ret = i915_gem_execbuffer_pin(dev, file,
3860                                       object_list, exec_list,
3861                                       args->buffer_count);
3862         if (ret)
3863                 goto err;
3864
3865         /* The objects are in their final locations, apply the relocations. */
3866         for (i = 0; i < args->buffer_count; i++) {
3867                 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3868                 obj->base.pending_read_domains = 0;
3869                 obj->base.pending_write_domain = 0;
3870                 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
3871                 if (ret)
3872                         goto err;
3873         }
3874
3875         /* Set the pending read domains for the batch buffer to COMMAND */
3876         batch_obj = object_list[args->buffer_count-1];
3877         if (batch_obj->pending_write_domain) {
3878                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3879                 ret = -EINVAL;
3880                 goto err;
3881         }
3882         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3883
3884         /* Sanity check the batch buffer */
3885         exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3886         ret = i915_gem_check_execbuffer(args, exec_offset);
3887         if (ret != 0) {
3888                 DRM_ERROR("execbuf with invalid offset/length\n");
3889                 goto err;
3890         }
3891
3892         ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
3893                                               object_list, args->buffer_count);
3894         if (ret)
3895                 goto err;
3896
3897 #if WATCH_COHERENCY
3898         for (i = 0; i < args->buffer_count; i++) {
3899                 i915_gem_object_check_coherency(object_list[i],
3900                                                 exec_list[i].handle);
3901         }
3902 #endif
3903
3904 #if WATCH_EXEC
3905         i915_gem_dump_object(batch_obj,
3906                               args->batch_len,
3907                               __func__,
3908                               ~0);
3909 #endif
3910
3911         /* Check for any pending flips. As we only maintain a flip queue depth
3912          * of 1, we can simply insert a WAIT for the next display flip prior
3913          * to executing the batch and avoid stalling the CPU.
3914          */
3915         flips = 0;
3916         for (i = 0; i < args->buffer_count; i++) {
3917                 if (object_list[i]->write_domain)
3918                         flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3919         }
3920         if (flips) {
3921                 int plane, flip_mask;
3922
3923                 for (plane = 0; flips >> plane; plane++) {
3924                         if (((flips >> plane) & 1) == 0)
3925                                 continue;
3926
3927                         if (plane)
3928                                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3929                         else
3930                                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3931
3932                         ret = intel_ring_begin(ring, 2);
3933                         if (ret)
3934                                 goto err;
3935
3936                         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
3937                         intel_ring_emit(ring, MI_NOOP);
3938                         intel_ring_advance(ring);
3939                 }
3940         }
3941
3942         /* Exec the batchbuffer */
3943         ret = ring->dispatch_execbuffer(ring, args, cliprects, exec_offset);
3944         if (ret) {
3945                 DRM_ERROR("dispatch failed %d\n", ret);
3946                 goto err;
3947         }
3948
3949         for (i = 0; i < args->buffer_count; i++) {
3950                 struct drm_gem_object *obj = object_list[i];
3951
3952                 obj->read_domains = obj->pending_read_domains;
3953                 obj->write_domain = obj->pending_write_domain;
3954
3955                 i915_gem_object_move_to_active(obj, ring);
3956                 if (obj->write_domain) {
3957                         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3958                         obj_priv->dirty = 1;
3959                         list_move_tail(&obj_priv->gpu_write_list,
3960                                        &ring->gpu_write_list);
3961                         intel_mark_busy(dev, obj);
3962                 }
3963
3964                 trace_i915_gem_object_change_domain(obj,
3965                                                     obj->read_domains,
3966                                                     obj->write_domain);
3967         }
3968
3969         /*
3970          * Ensure that the commands in the batch buffer are
3971          * finished before the interrupt fires
3972          */
3973         i915_retire_commands(dev, ring);
3974
3975         if (i915_add_request(dev, file, request, ring))
3976                 ring->outstanding_lazy_request = true;
3977         else
3978                 request = NULL;
3979
3980 err:
3981         for (i = 0; i < args->buffer_count; i++) {
3982                 if (object_list[i] == NULL)
3983                     break;
3984
3985                 to_intel_bo(object_list[i])->in_execbuffer = false;
3986                 drm_gem_object_unreference(object_list[i]);
3987         }
3988
3989         mutex_unlock(&dev->struct_mutex);
3990
3991 pre_mutex_err:
3992         drm_free_large(object_list);
3993         kfree(cliprects);
3994         kfree(request);
3995
3996         return ret;
3997 }
3998
3999 /*
4000  * Legacy execbuffer just creates an exec2 list from the original exec object
4001  * list array and passes it to the real function.
4002  */
4003 int
4004 i915_gem_execbuffer(struct drm_device *dev, void *data,
4005                     struct drm_file *file_priv)
4006 {
4007         struct drm_i915_gem_execbuffer *args = data;
4008         struct drm_i915_gem_execbuffer2 exec2;
4009         struct drm_i915_gem_exec_object *exec_list = NULL;
4010         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4011         int ret, i;
4012
4013 #if WATCH_EXEC
4014         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4015                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4016 #endif
4017
4018         if (args->buffer_count < 1) {
4019                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4020                 return -EINVAL;
4021         }
4022
4023         /* Copy in the exec list from userland */
4024         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4025         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4026         if (exec_list == NULL || exec2_list == NULL) {
4027                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4028                           args->buffer_count);
4029                 drm_free_large(exec_list);
4030                 drm_free_large(exec2_list);
4031                 return -ENOMEM;
4032         }
4033         ret = copy_from_user(exec_list,
4034                              (struct drm_i915_relocation_entry __user *)
4035                              (uintptr_t) args->buffers_ptr,
4036                              sizeof(*exec_list) * args->buffer_count);
4037         if (ret != 0) {
4038                 DRM_ERROR("copy %d exec entries failed %d\n",
4039                           args->buffer_count, ret);
4040                 drm_free_large(exec_list);
4041                 drm_free_large(exec2_list);
4042                 return -EFAULT;
4043         }
4044
4045         for (i = 0; i < args->buffer_count; i++) {
4046                 exec2_list[i].handle = exec_list[i].handle;
4047                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4048                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4049                 exec2_list[i].alignment = exec_list[i].alignment;
4050                 exec2_list[i].offset = exec_list[i].offset;
4051                 if (INTEL_INFO(dev)->gen < 4)
4052                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4053                 else
4054                         exec2_list[i].flags = 0;
4055         }
4056
4057         exec2.buffers_ptr = args->buffers_ptr;
4058         exec2.buffer_count = args->buffer_count;
4059         exec2.batch_start_offset = args->batch_start_offset;
4060         exec2.batch_len = args->batch_len;
4061         exec2.DR1 = args->DR1;
4062         exec2.DR4 = args->DR4;
4063         exec2.num_cliprects = args->num_cliprects;
4064         exec2.cliprects_ptr = args->cliprects_ptr;
4065         exec2.flags = I915_EXEC_RENDER;
4066
4067         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4068         if (!ret) {
4069                 /* Copy the new buffer offsets back to the user's exec list. */
4070                 for (i = 0; i < args->buffer_count; i++)
4071                         exec_list[i].offset = exec2_list[i].offset;
4072                 /* ... and back out to userspace */
4073                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4074                                    (uintptr_t) args->buffers_ptr,
4075                                    exec_list,
4076                                    sizeof(*exec_list) * args->buffer_count);
4077                 if (ret) {
4078                         ret = -EFAULT;
4079                         DRM_ERROR("failed to copy %d exec entries "
4080                                   "back to user (%d)\n",
4081                                   args->buffer_count, ret);
4082                 }
4083         }
4084
4085         drm_free_large(exec_list);
4086         drm_free_large(exec2_list);
4087         return ret;
4088 }
4089
4090 int
4091 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4092                      struct drm_file *file_priv)
4093 {
4094         struct drm_i915_gem_execbuffer2 *args = data;
4095         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4096         int ret;
4097
4098 #if WATCH_EXEC
4099         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4100                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4101 #endif
4102
4103         if (args->buffer_count < 1) {
4104                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4105                 return -EINVAL;
4106         }
4107
4108         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4109         if (exec2_list == NULL) {
4110                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4111                           args->buffer_count);
4112                 return -ENOMEM;
4113         }
4114         ret = copy_from_user(exec2_list,
4115                              (struct drm_i915_relocation_entry __user *)
4116                              (uintptr_t) args->buffers_ptr,
4117                              sizeof(*exec2_list) * args->buffer_count);
4118         if (ret != 0) {
4119                 DRM_ERROR("copy %d exec entries failed %d\n",
4120                           args->buffer_count, ret);
4121                 drm_free_large(exec2_list);
4122                 return -EFAULT;
4123         }
4124
4125         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4126         if (!ret) {
4127                 /* Copy the new buffer offsets back to the user's exec list. */
4128                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4129                                    (uintptr_t) args->buffers_ptr,
4130                                    exec2_list,
4131                                    sizeof(*exec2_list) * args->buffer_count);
4132                 if (ret) {
4133                         ret = -EFAULT;
4134                         DRM_ERROR("failed to copy %d exec entries "
4135                                   "back to user (%d)\n",
4136                                   args->buffer_count, ret);
4137                 }
4138         }
4139
4140         drm_free_large(exec2_list);
4141         return ret;
4142 }
4143
4144 int
4145 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment,
4146                     bool map_and_fenceable)
4147 {
4148         struct drm_device *dev = obj->dev;
4149         struct drm_i915_private *dev_priv = dev->dev_private;
4150         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4151         int ret;
4152
4153         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4154         BUG_ON(map_and_fenceable && !map_and_fenceable);
4155         WARN_ON(i915_verify_lists(dev));
4156
4157         if (obj_priv->gtt_space != NULL) {
4158                 if ((alignment && obj_priv->gtt_offset & (alignment - 1)) ||
4159                     (map_and_fenceable && !obj_priv->map_and_fenceable)) {
4160                         WARN(obj_priv->pin_count,
4161                              "bo is already pinned with incorrect alignment:"
4162                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
4163                              " obj->map_and_fenceable=%d\n",
4164                              obj_priv->gtt_offset, alignment,
4165                              map_and_fenceable,
4166                              obj_priv->map_and_fenceable);
4167                         ret = i915_gem_object_unbind(obj);
4168                         if (ret)
4169                                 return ret;
4170                 }
4171         }
4172
4173         if (obj_priv->gtt_space == NULL) {
4174                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
4175                                                   map_and_fenceable);
4176                 if (ret)
4177                         return ret;
4178         }
4179
4180         if (obj_priv->pin_count++ == 0) {
4181                 i915_gem_info_add_pin(dev_priv, obj_priv, map_and_fenceable);
4182                 if (!obj_priv->active)
4183                         list_move_tail(&obj_priv->mm_list,
4184                                        &dev_priv->mm.pinned_list);
4185         }
4186         BUG_ON(!obj_priv->pin_mappable && map_and_fenceable);
4187
4188         WARN_ON(i915_verify_lists(dev));
4189         return 0;
4190 }
4191
4192 void
4193 i915_gem_object_unpin(struct drm_gem_object *obj)
4194 {
4195         struct drm_device *dev = obj->dev;
4196         drm_i915_private_t *dev_priv = dev->dev_private;
4197         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4198
4199         WARN_ON(i915_verify_lists(dev));
4200         BUG_ON(obj_priv->pin_count == 0);
4201         BUG_ON(obj_priv->gtt_space == NULL);
4202
4203         if (--obj_priv->pin_count == 0) {
4204                 if (!obj_priv->active)
4205                         list_move_tail(&obj_priv->mm_list,
4206                                        &dev_priv->mm.inactive_list);
4207                 i915_gem_info_remove_pin(dev_priv, obj_priv);
4208         }
4209         WARN_ON(i915_verify_lists(dev));
4210 }
4211
4212 int
4213 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4214                    struct drm_file *file_priv)
4215 {
4216         struct drm_i915_gem_pin *args = data;
4217         struct drm_gem_object *obj;
4218         struct drm_i915_gem_object *obj_priv;
4219         int ret;
4220
4221         ret = i915_mutex_lock_interruptible(dev);
4222         if (ret)
4223                 return ret;
4224
4225         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4226         if (obj == NULL) {
4227                 ret = -ENOENT;
4228                 goto unlock;
4229         }
4230         obj_priv = to_intel_bo(obj);
4231
4232         if (obj_priv->madv != I915_MADV_WILLNEED) {
4233                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4234                 ret = -EINVAL;
4235                 goto out;
4236         }
4237
4238         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4239                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4240                           args->handle);
4241                 ret = -EINVAL;
4242                 goto out;
4243         }
4244
4245         obj_priv->user_pin_count++;
4246         obj_priv->pin_filp = file_priv;
4247         if (obj_priv->user_pin_count == 1) {
4248                 ret = i915_gem_object_pin(obj, args->alignment, true);
4249                 if (ret)
4250                         goto out;
4251         }
4252
4253         /* XXX - flush the CPU caches for pinned objects
4254          * as the X server doesn't manage domains yet
4255          */
4256         i915_gem_object_flush_cpu_write_domain(obj);
4257         args->offset = obj_priv->gtt_offset;
4258 out:
4259         drm_gem_object_unreference(obj);
4260 unlock:
4261         mutex_unlock(&dev->struct_mutex);
4262         return ret;
4263 }
4264
4265 int
4266 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4267                      struct drm_file *file_priv)
4268 {
4269         struct drm_i915_gem_pin *args = data;
4270         struct drm_gem_object *obj;
4271         struct drm_i915_gem_object *obj_priv;
4272         int ret;
4273
4274         ret = i915_mutex_lock_interruptible(dev);
4275         if (ret)
4276                 return ret;
4277
4278         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4279         if (obj == NULL) {
4280                 ret = -ENOENT;
4281                 goto unlock;
4282         }
4283         obj_priv = to_intel_bo(obj);
4284
4285         if (obj_priv->pin_filp != file_priv) {
4286                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4287                           args->handle);
4288                 ret = -EINVAL;
4289                 goto out;
4290         }
4291         obj_priv->user_pin_count--;
4292         if (obj_priv->user_pin_count == 0) {
4293                 obj_priv->pin_filp = NULL;
4294                 i915_gem_object_unpin(obj);
4295         }
4296
4297 out:
4298         drm_gem_object_unreference(obj);
4299 unlock:
4300         mutex_unlock(&dev->struct_mutex);
4301         return ret;
4302 }
4303
4304 int
4305 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4306                     struct drm_file *file_priv)
4307 {
4308         struct drm_i915_gem_busy *args = data;
4309         struct drm_gem_object *obj;
4310         struct drm_i915_gem_object *obj_priv;
4311         int ret;
4312
4313         ret = i915_mutex_lock_interruptible(dev);
4314         if (ret)
4315                 return ret;
4316
4317         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4318         if (obj == NULL) {
4319                 ret = -ENOENT;
4320                 goto unlock;
4321         }
4322         obj_priv = to_intel_bo(obj);
4323
4324         /* Count all active objects as busy, even if they are currently not used
4325          * by the gpu. Users of this interface expect objects to eventually
4326          * become non-busy without any further actions, therefore emit any
4327          * necessary flushes here.
4328          */
4329         args->busy = obj_priv->active;
4330         if (args->busy) {
4331                 /* Unconditionally flush objects, even when the gpu still uses this
4332                  * object. Userspace calling this function indicates that it wants to
4333                  * use this buffer rather sooner than later, so issuing the required
4334                  * flush earlier is beneficial.
4335                  */
4336                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4337                         i915_gem_flush_ring(dev, file_priv,
4338                                             obj_priv->ring,
4339                                             0, obj->write_domain);
4340
4341                 /* Update the active list for the hardware's current position.
4342                  * Otherwise this only updates on a delayed timer or when irqs
4343                  * are actually unmasked, and our working set ends up being
4344                  * larger than required.
4345                  */
4346                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4347
4348                 args->busy = obj_priv->active;
4349         }
4350
4351         drm_gem_object_unreference(obj);
4352 unlock:
4353         mutex_unlock(&dev->struct_mutex);
4354         return ret;
4355 }
4356
4357 int
4358 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4359                         struct drm_file *file_priv)
4360 {
4361     return i915_gem_ring_throttle(dev, file_priv);
4362 }
4363
4364 int
4365 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4366                        struct drm_file *file_priv)
4367 {
4368         struct drm_i915_gem_madvise *args = data;
4369         struct drm_gem_object *obj;
4370         struct drm_i915_gem_object *obj_priv;
4371         int ret;
4372
4373         switch (args->madv) {
4374         case I915_MADV_DONTNEED:
4375         case I915_MADV_WILLNEED:
4376             break;
4377         default:
4378             return -EINVAL;
4379         }
4380
4381         ret = i915_mutex_lock_interruptible(dev);
4382         if (ret)
4383                 return ret;
4384
4385         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4386         if (obj == NULL) {
4387                 ret = -ENOENT;
4388                 goto unlock;
4389         }
4390         obj_priv = to_intel_bo(obj);
4391
4392         if (obj_priv->pin_count) {
4393                 ret = -EINVAL;
4394                 goto out;
4395         }
4396
4397         if (obj_priv->madv != __I915_MADV_PURGED)
4398                 obj_priv->madv = args->madv;
4399
4400         /* if the object is no longer bound, discard its backing storage */
4401         if (i915_gem_object_is_purgeable(obj_priv) &&
4402             obj_priv->gtt_space == NULL)
4403                 i915_gem_object_truncate(obj);
4404
4405         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4406
4407 out:
4408         drm_gem_object_unreference(obj);
4409 unlock:
4410         mutex_unlock(&dev->struct_mutex);
4411         return ret;
4412 }
4413
4414 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4415                                               size_t size)
4416 {
4417         struct drm_i915_private *dev_priv = dev->dev_private;
4418         struct drm_i915_gem_object *obj;
4419
4420         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4421         if (obj == NULL)
4422                 return NULL;
4423
4424         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4425                 kfree(obj);
4426                 return NULL;
4427         }
4428
4429         i915_gem_info_add_obj(dev_priv, size);
4430
4431         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4432         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4433
4434         obj->agp_type = AGP_USER_MEMORY;
4435         obj->base.driver_private = NULL;
4436         obj->fence_reg = I915_FENCE_REG_NONE;
4437         INIT_LIST_HEAD(&obj->mm_list);
4438         INIT_LIST_HEAD(&obj->ring_list);
4439         INIT_LIST_HEAD(&obj->gpu_write_list);
4440         obj->madv = I915_MADV_WILLNEED;
4441         /* Avoid an unnecessary call to unbind on the first bind. */
4442         obj->map_and_fenceable = true;
4443
4444         return &obj->base;
4445 }
4446
4447 int i915_gem_init_object(struct drm_gem_object *obj)
4448 {
4449         BUG();
4450
4451         return 0;
4452 }
4453
4454 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4455 {
4456         struct drm_device *dev = obj->dev;
4457         drm_i915_private_t *dev_priv = dev->dev_private;
4458         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4459         int ret;
4460
4461         ret = i915_gem_object_unbind(obj);
4462         if (ret == -ERESTARTSYS) {
4463                 list_move(&obj_priv->mm_list,
4464                           &dev_priv->mm.deferred_free_list);
4465                 return;
4466         }
4467
4468         if (obj->map_list.map)
4469                 i915_gem_free_mmap_offset(obj);
4470
4471         drm_gem_object_release(obj);
4472         i915_gem_info_remove_obj(dev_priv, obj->size);
4473
4474         kfree(obj_priv->page_cpu_valid);
4475         kfree(obj_priv->bit_17);
4476         kfree(obj_priv);
4477 }
4478
4479 void i915_gem_free_object(struct drm_gem_object *obj)
4480 {
4481         struct drm_device *dev = obj->dev;
4482         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4483
4484         trace_i915_gem_object_destroy(obj);
4485
4486         while (obj_priv->pin_count > 0)
4487                 i915_gem_object_unpin(obj);
4488
4489         if (obj_priv->phys_obj)
4490                 i915_gem_detach_phys_object(dev, obj);
4491
4492         i915_gem_free_object_tail(obj);
4493 }
4494
4495 int
4496 i915_gem_idle(struct drm_device *dev)
4497 {
4498         drm_i915_private_t *dev_priv = dev->dev_private;
4499         int ret;
4500
4501         mutex_lock(&dev->struct_mutex);
4502
4503         if (dev_priv->mm.suspended) {
4504                 mutex_unlock(&dev->struct_mutex);
4505                 return 0;
4506         }
4507
4508         ret = i915_gpu_idle(dev);
4509         if (ret) {
4510                 mutex_unlock(&dev->struct_mutex);
4511                 return ret;
4512         }
4513
4514         /* Under UMS, be paranoid and evict. */
4515         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4516                 ret = i915_gem_evict_inactive(dev, false);
4517                 if (ret) {
4518                         mutex_unlock(&dev->struct_mutex);
4519                         return ret;
4520                 }
4521         }
4522
4523         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4524          * We need to replace this with a semaphore, or something.
4525          * And not confound mm.suspended!
4526          */
4527         dev_priv->mm.suspended = 1;
4528         del_timer_sync(&dev_priv->hangcheck_timer);
4529
4530         i915_kernel_lost_context(dev);
4531         i915_gem_cleanup_ringbuffer(dev);
4532
4533         mutex_unlock(&dev->struct_mutex);
4534
4535         /* Cancel the retire work handler, which should be idle now. */
4536         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4537
4538         return 0;
4539 }
4540
4541 /*
4542  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4543  * over cache flushing.
4544  */
4545 static int
4546 i915_gem_init_pipe_control(struct drm_device *dev)
4547 {
4548         drm_i915_private_t *dev_priv = dev->dev_private;
4549         struct drm_gem_object *obj;
4550         struct drm_i915_gem_object *obj_priv;
4551         int ret;
4552
4553         obj = i915_gem_alloc_object(dev, 4096);
4554         if (obj == NULL) {
4555                 DRM_ERROR("Failed to allocate seqno page\n");
4556                 ret = -ENOMEM;
4557                 goto err;
4558         }
4559         obj_priv = to_intel_bo(obj);
4560         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4561
4562         ret = i915_gem_object_pin(obj, 4096, true);
4563         if (ret)
4564                 goto err_unref;
4565
4566         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4567         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4568         if (dev_priv->seqno_page == NULL)
4569                 goto err_unpin;
4570
4571         dev_priv->seqno_obj = obj;
4572         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4573
4574         return 0;
4575
4576 err_unpin:
4577         i915_gem_object_unpin(obj);
4578 err_unref:
4579         drm_gem_object_unreference(obj);
4580 err:
4581         return ret;
4582 }
4583
4584
4585 static void
4586 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4587 {
4588         drm_i915_private_t *dev_priv = dev->dev_private;
4589         struct drm_gem_object *obj;
4590         struct drm_i915_gem_object *obj_priv;
4591
4592         obj = dev_priv->seqno_obj;
4593         obj_priv = to_intel_bo(obj);
4594         kunmap(obj_priv->pages[0]);
4595         i915_gem_object_unpin(obj);
4596         drm_gem_object_unreference(obj);
4597         dev_priv->seqno_obj = NULL;
4598
4599         dev_priv->seqno_page = NULL;
4600 }
4601
4602 int
4603 i915_gem_init_ringbuffer(struct drm_device *dev)
4604 {
4605         drm_i915_private_t *dev_priv = dev->dev_private;
4606         int ret;
4607
4608         if (HAS_PIPE_CONTROL(dev)) {
4609                 ret = i915_gem_init_pipe_control(dev);
4610                 if (ret)
4611                         return ret;
4612         }
4613
4614         ret = intel_init_render_ring_buffer(dev);
4615         if (ret)
4616                 goto cleanup_pipe_control;
4617
4618         if (HAS_BSD(dev)) {
4619                 ret = intel_init_bsd_ring_buffer(dev);
4620                 if (ret)
4621                         goto cleanup_render_ring;
4622         }
4623
4624         if (HAS_BLT(dev)) {
4625                 ret = intel_init_blt_ring_buffer(dev);
4626                 if (ret)
4627                         goto cleanup_bsd_ring;
4628         }
4629
4630         dev_priv->next_seqno = 1;
4631
4632         return 0;
4633
4634 cleanup_bsd_ring:
4635         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4636 cleanup_render_ring:
4637         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4638 cleanup_pipe_control:
4639         if (HAS_PIPE_CONTROL(dev))
4640                 i915_gem_cleanup_pipe_control(dev);
4641         return ret;
4642 }
4643
4644 void
4645 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4646 {
4647         drm_i915_private_t *dev_priv = dev->dev_private;
4648
4649         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4650         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4651         intel_cleanup_ring_buffer(&dev_priv->blt_ring);
4652         if (HAS_PIPE_CONTROL(dev))
4653                 i915_gem_cleanup_pipe_control(dev);
4654 }
4655
4656 int
4657 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4658                        struct drm_file *file_priv)
4659 {
4660         drm_i915_private_t *dev_priv = dev->dev_private;
4661         int ret;
4662
4663         if (drm_core_check_feature(dev, DRIVER_MODESET))
4664                 return 0;
4665
4666         if (atomic_read(&dev_priv->mm.wedged)) {
4667                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4668                 atomic_set(&dev_priv->mm.wedged, 0);
4669         }
4670
4671         mutex_lock(&dev->struct_mutex);
4672         dev_priv->mm.suspended = 0;
4673
4674         ret = i915_gem_init_ringbuffer(dev);
4675         if (ret != 0) {
4676                 mutex_unlock(&dev->struct_mutex);
4677                 return ret;
4678         }
4679
4680         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4681         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4682         BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4683         BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4684         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4685         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4686         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4687         BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4688         BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4689         mutex_unlock(&dev->struct_mutex);
4690
4691         ret = drm_irq_install(dev);
4692         if (ret)
4693                 goto cleanup_ringbuffer;
4694
4695         return 0;
4696
4697 cleanup_ringbuffer:
4698         mutex_lock(&dev->struct_mutex);
4699         i915_gem_cleanup_ringbuffer(dev);
4700         dev_priv->mm.suspended = 1;
4701         mutex_unlock(&dev->struct_mutex);
4702
4703         return ret;
4704 }
4705
4706 int
4707 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4708                        struct drm_file *file_priv)
4709 {
4710         if (drm_core_check_feature(dev, DRIVER_MODESET))
4711                 return 0;
4712
4713         drm_irq_uninstall(dev);
4714         return i915_gem_idle(dev);
4715 }
4716
4717 void
4718 i915_gem_lastclose(struct drm_device *dev)
4719 {
4720         int ret;
4721
4722         if (drm_core_check_feature(dev, DRIVER_MODESET))
4723                 return;
4724
4725         ret = i915_gem_idle(dev);
4726         if (ret)
4727                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4728 }
4729
4730 static void
4731 init_ring_lists(struct intel_ring_buffer *ring)
4732 {
4733         INIT_LIST_HEAD(&ring->active_list);
4734         INIT_LIST_HEAD(&ring->request_list);
4735         INIT_LIST_HEAD(&ring->gpu_write_list);
4736 }
4737
4738 void
4739 i915_gem_load(struct drm_device *dev)
4740 {
4741         int i;
4742         drm_i915_private_t *dev_priv = dev->dev_private;
4743
4744         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4745         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4746         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4747         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4748         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4749         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4750         init_ring_lists(&dev_priv->render_ring);
4751         init_ring_lists(&dev_priv->bsd_ring);
4752         init_ring_lists(&dev_priv->blt_ring);
4753         for (i = 0; i < 16; i++)
4754                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4755         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4756                           i915_gem_retire_work_handler);
4757         init_completion(&dev_priv->error_completion);
4758
4759         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4760         if (IS_GEN3(dev)) {
4761                 u32 tmp = I915_READ(MI_ARB_STATE);
4762                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4763                         /* arb state is a masked write, so set bit + bit in mask */
4764                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4765                         I915_WRITE(MI_ARB_STATE, tmp);
4766                 }
4767         }
4768
4769         /* Old X drivers will take 0-2 for front, back, depth buffers */
4770         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4771                 dev_priv->fence_reg_start = 3;
4772
4773         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4774                 dev_priv->num_fence_regs = 16;
4775         else
4776                 dev_priv->num_fence_regs = 8;
4777
4778         /* Initialize fence registers to zero */
4779         switch (INTEL_INFO(dev)->gen) {
4780         case 6:
4781                 for (i = 0; i < 16; i++)
4782                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4783                 break;
4784         case 5:
4785         case 4:
4786                 for (i = 0; i < 16; i++)
4787                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4788                 break;
4789         case 3:
4790                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4791                         for (i = 0; i < 8; i++)
4792                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4793         case 2:
4794                 for (i = 0; i < 8; i++)
4795                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4796                 break;
4797         }
4798         i915_gem_detect_bit_6_swizzle(dev);
4799         init_waitqueue_head(&dev_priv->pending_flip_queue);
4800
4801         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4802         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4803         register_shrinker(&dev_priv->mm.inactive_shrinker);
4804 }
4805
4806 /*
4807  * Create a physically contiguous memory object for this object
4808  * e.g. for cursor + overlay regs
4809  */
4810 static int i915_gem_init_phys_object(struct drm_device *dev,
4811                                      int id, int size, int align)
4812 {
4813         drm_i915_private_t *dev_priv = dev->dev_private;
4814         struct drm_i915_gem_phys_object *phys_obj;
4815         int ret;
4816
4817         if (dev_priv->mm.phys_objs[id - 1] || !size)
4818                 return 0;
4819
4820         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4821         if (!phys_obj)
4822                 return -ENOMEM;
4823
4824         phys_obj->id = id;
4825
4826         phys_obj->handle = drm_pci_alloc(dev, size, align);
4827         if (!phys_obj->handle) {
4828                 ret = -ENOMEM;
4829                 goto kfree_obj;
4830         }
4831 #ifdef CONFIG_X86
4832         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4833 #endif
4834
4835         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4836
4837         return 0;
4838 kfree_obj:
4839         kfree(phys_obj);
4840         return ret;
4841 }
4842
4843 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4844 {
4845         drm_i915_private_t *dev_priv = dev->dev_private;
4846         struct drm_i915_gem_phys_object *phys_obj;
4847
4848         if (!dev_priv->mm.phys_objs[id - 1])
4849                 return;
4850
4851         phys_obj = dev_priv->mm.phys_objs[id - 1];
4852         if (phys_obj->cur_obj) {
4853                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4854         }
4855
4856 #ifdef CONFIG_X86
4857         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4858 #endif
4859         drm_pci_free(dev, phys_obj->handle);
4860         kfree(phys_obj);
4861         dev_priv->mm.phys_objs[id - 1] = NULL;
4862 }
4863
4864 void i915_gem_free_all_phys_object(struct drm_device *dev)
4865 {
4866         int i;
4867
4868         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4869                 i915_gem_free_phys_object(dev, i);
4870 }
4871
4872 void i915_gem_detach_phys_object(struct drm_device *dev,
4873                                  struct drm_gem_object *obj)
4874 {
4875         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4876         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4877         char *vaddr;
4878         int i;
4879         int page_count;
4880
4881         if (!obj_priv->phys_obj)
4882                 return;
4883         vaddr = obj_priv->phys_obj->handle->vaddr;
4884
4885         page_count = obj->size / PAGE_SIZE;
4886
4887         for (i = 0; i < page_count; i++) {
4888                 struct page *page = read_cache_page_gfp(mapping, i,
4889                                                         GFP_HIGHUSER | __GFP_RECLAIMABLE);
4890                 if (!IS_ERR(page)) {
4891                         char *dst = kmap_atomic(page);
4892                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4893                         kunmap_atomic(dst);
4894
4895                         drm_clflush_pages(&page, 1);
4896
4897                         set_page_dirty(page);
4898                         mark_page_accessed(page);
4899                         page_cache_release(page);
4900                 }
4901         }
4902         drm_agp_chipset_flush(dev);
4903
4904         obj_priv->phys_obj->cur_obj = NULL;
4905         obj_priv->phys_obj = NULL;
4906 }
4907
4908 int
4909 i915_gem_attach_phys_object(struct drm_device *dev,
4910                             struct drm_gem_object *obj,
4911                             int id,
4912                             int align)
4913 {
4914         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4915         drm_i915_private_t *dev_priv = dev->dev_private;
4916         struct drm_i915_gem_object *obj_priv;
4917         int ret = 0;
4918         int page_count;
4919         int i;
4920
4921         if (id > I915_MAX_PHYS_OBJECT)
4922                 return -EINVAL;
4923
4924         obj_priv = to_intel_bo(obj);
4925
4926         if (obj_priv->phys_obj) {
4927                 if (obj_priv->phys_obj->id == id)
4928                         return 0;
4929                 i915_gem_detach_phys_object(dev, obj);
4930         }
4931
4932         /* create a new object */
4933         if (!dev_priv->mm.phys_objs[id - 1]) {
4934                 ret = i915_gem_init_phys_object(dev, id,
4935                                                 obj->size, align);
4936                 if (ret) {
4937                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4938                         return ret;
4939                 }
4940         }
4941
4942         /* bind to the object */
4943         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4944         obj_priv->phys_obj->cur_obj = obj;
4945
4946         page_count = obj->size / PAGE_SIZE;
4947
4948         for (i = 0; i < page_count; i++) {
4949                 struct page *page;
4950                 char *dst, *src;
4951
4952                 page = read_cache_page_gfp(mapping, i,
4953                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
4954                 if (IS_ERR(page))
4955                         return PTR_ERR(page);
4956
4957                 src = kmap_atomic(page);
4958                 dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4959                 memcpy(dst, src, PAGE_SIZE);
4960                 kunmap_atomic(src);
4961
4962                 mark_page_accessed(page);
4963                 page_cache_release(page);
4964         }
4965
4966         return 0;
4967 }
4968
4969 static int
4970 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4971                      struct drm_i915_gem_pwrite *args,
4972                      struct drm_file *file_priv)
4973 {
4974         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4975         void *obj_addr;
4976         int ret;
4977         char __user *user_data;
4978
4979         user_data = (char __user *) (uintptr_t) args->data_ptr;
4980         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4981
4982         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4983         ret = copy_from_user(obj_addr, user_data, args->size);
4984         if (ret)
4985                 return -EFAULT;
4986
4987         drm_agp_chipset_flush(dev);
4988         return 0;
4989 }
4990
4991 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4992 {
4993         struct drm_i915_file_private *file_priv = file->driver_priv;
4994
4995         /* Clean up our request list when the client is going away, so that
4996          * later retire_requests won't dereference our soon-to-be-gone
4997          * file_priv.
4998          */
4999         spin_lock(&file_priv->mm.lock);
5000         while (!list_empty(&file_priv->mm.request_list)) {
5001                 struct drm_i915_gem_request *request;
5002
5003                 request = list_first_entry(&file_priv->mm.request_list,
5004                                            struct drm_i915_gem_request,
5005                                            client_list);
5006                 list_del(&request->client_list);
5007                 request->file_priv = NULL;
5008         }
5009         spin_unlock(&file_priv->mm.lock);
5010 }
5011
5012 static int
5013 i915_gpu_is_active(struct drm_device *dev)
5014 {
5015         drm_i915_private_t *dev_priv = dev->dev_private;
5016         int lists_empty;
5017
5018         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5019                       list_empty(&dev_priv->mm.active_list);
5020
5021         return !lists_empty;
5022 }
5023
5024 static int
5025 i915_gem_inactive_shrink(struct shrinker *shrinker,
5026                          int nr_to_scan,
5027                          gfp_t gfp_mask)
5028 {
5029         struct drm_i915_private *dev_priv =
5030                 container_of(shrinker,
5031                              struct drm_i915_private,
5032                              mm.inactive_shrinker);
5033         struct drm_device *dev = dev_priv->dev;
5034         struct drm_i915_gem_object *obj, *next;
5035         int cnt;
5036
5037         if (!mutex_trylock(&dev->struct_mutex))
5038                 return 0;
5039
5040         /* "fast-path" to count number of available objects */
5041         if (nr_to_scan == 0) {
5042                 cnt = 0;
5043                 list_for_each_entry(obj,
5044                                     &dev_priv->mm.inactive_list,
5045                                     mm_list)
5046                         cnt++;
5047                 mutex_unlock(&dev->struct_mutex);
5048                 return cnt / 100 * sysctl_vfs_cache_pressure;
5049         }
5050
5051 rescan:
5052         /* first scan for clean buffers */
5053         i915_gem_retire_requests(dev);
5054
5055         list_for_each_entry_safe(obj, next,
5056                                  &dev_priv->mm.inactive_list,
5057                                  mm_list) {
5058                 if (i915_gem_object_is_purgeable(obj)) {
5059                         i915_gem_object_unbind(&obj->base);
5060                         if (--nr_to_scan == 0)
5061                                 break;
5062                 }
5063         }
5064
5065         /* second pass, evict/count anything still on the inactive list */
5066         cnt = 0;
5067         list_for_each_entry_safe(obj, next,
5068                                  &dev_priv->mm.inactive_list,
5069                                  mm_list) {
5070                 if (nr_to_scan) {
5071                         i915_gem_object_unbind(&obj->base);
5072                         nr_to_scan--;
5073                 } else
5074                         cnt++;
5075         }
5076
5077         if (nr_to_scan && i915_gpu_is_active(dev)) {
5078                 /*
5079                  * We are desperate for pages, so as a last resort, wait
5080                  * for the GPU to finish and discard whatever we can.
5081                  * This has a dramatic impact to reduce the number of
5082                  * OOM-killer events whilst running the GPU aggressively.
5083                  */
5084                 if (i915_gpu_idle(dev) == 0)
5085                         goto rescan;
5086         }
5087         mutex_unlock(&dev->struct_mutex);
5088         return cnt / 100 * sysctl_vfs_cache_pressure;
5089 }