drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 struct change_domains {
  40         uint32_t invalidate_domains;
  41         uint32_t flush_domains;
  42         uint32_t flush_rings;
  43 };
  44
  45 static uint32_t i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv);
  46 static uint32_t i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv);
  47
  48 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
  49                                                   bool pipelined);
  50 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  51 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  52 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  53                                              int write);
  54 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  55                                                      uint64_t offset,
  56                                                      uint64_t size);
  57 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  58 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
  59                                           bool interruptible);
  60 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  61                                        unsigned alignment,
  62                                        bool map_and_fenceable);
  63 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  64 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  65                                 struct drm_i915_gem_pwrite *args,
  66                                 struct drm_file *file_priv);
  67 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  68
  69 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  70                                     int nr_to_scan,
  71                                     gfp_t gfp_mask);
  72
  73
  74 /* some bookkeeping */
  75 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  76                                   size_t size)
  77 {
  78         dev_priv->mm.object_count++;
  79         dev_priv->mm.object_memory += size;
  80 }
  81
  82 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  83                                      size_t size)
  84 {
  85         dev_priv->mm.object_count--;
  86         dev_priv->mm.object_memory -= size;
  87 }
  88
  89 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
  90                                   struct drm_i915_gem_object *obj)
  91 {
  92         dev_priv->mm.gtt_count++;
  93         dev_priv->mm.gtt_memory += obj->gtt_space->size;
  94         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
  95                 dev_priv->mm.mappable_gtt_used +=
  96                         min_t(size_t, obj->gtt_space->size,
  97                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
  98         }
  99 }
 100
 101 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
 102                                      struct drm_i915_gem_object *obj)
 103 {
 104         dev_priv->mm.gtt_count--;
 105         dev_priv->mm.gtt_memory -= obj->gtt_space->size;
 106         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
 107                 dev_priv->mm.mappable_gtt_used -=
 108                         min_t(size_t, obj->gtt_space->size,
 109                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
 110         }
 111 }
 112
 113 /**
 114  * Update the mappable working set counters. Call _only_ when there is a change
 115  * in one of (pin|fault)_mappable and update *_mappable _before_ calling.
 116  * @mappable: new state the changed mappable flag (either pin_ or fault_).
 117  */
 118 static void
 119 i915_gem_info_update_mappable(struct drm_i915_private *dev_priv,
 120                               struct drm_i915_gem_object *obj,
 121                               bool mappable)
 122 {
 123         if (mappable) {
 124                 if (obj->pin_mappable && obj->fault_mappable)
 125                         /* Combined state was already mappable. */
 126                         return;
 127                 dev_priv->mm.gtt_mappable_count++;
 128                 dev_priv->mm.gtt_mappable_memory += obj->gtt_space->size;
 129         } else {
 130                 if (obj->pin_mappable || obj->fault_mappable)
 131                         /* Combined state still mappable. */
 132                         return;
 133                 dev_priv->mm.gtt_mappable_count--;
 134                 dev_priv->mm.gtt_mappable_memory -= obj->gtt_space->size;
 135         }
 136 }
 137
 138 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
 139                                   struct drm_i915_gem_object *obj,
 140                                   bool mappable)
 141 {
 142         dev_priv->mm.pin_count++;
 143         dev_priv->mm.pin_memory += obj->gtt_space->size;
 144         if (mappable) {
 145                 obj->pin_mappable = true;
 146                 i915_gem_info_update_mappable(dev_priv, obj, true);
 147         }
 148 }
 149
 150 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
 151                                      struct drm_i915_gem_object *obj)
 152 {
 153         dev_priv->mm.pin_count--;
 154         dev_priv->mm.pin_memory -= obj->gtt_space->size;
 155         if (obj->pin_mappable) {
 156                 obj->pin_mappable = false;
 157                 i915_gem_info_update_mappable(dev_priv, obj, false);
 158         }
 159 }
 160
 161 int
 162 i915_gem_check_is_wedged(struct drm_device *dev)
 163 {
 164         struct drm_i915_private *dev_priv = dev->dev_private;
 165         struct completion *x = &dev_priv->error_completion;
 166         unsigned long flags;
 167         int ret;
 168
 169         if (!atomic_read(&dev_priv->mm.wedged))
 170                 return 0;
 171
 172         ret = wait_for_completion_interruptible(x);
 173         if (ret)
 174                 return ret;
 175
 176         /* Success, we reset the GPU! */
 177         if (!atomic_read(&dev_priv->mm.wedged))
 178                 return 0;
 179
 180         /* GPU is hung, bump the completion count to account for
 181          * the token we just consumed so that we never hit zero and
 182          * end up waiting upon a subsequent completion event that
 183          * will never happen.
 184          */
 185         spin_lock_irqsave(&x->wait.lock, flags);
 186         x->done++;
 187         spin_unlock_irqrestore(&x->wait.lock, flags);
 188         return -EIO;
 189 }
 190
 191 static int i915_mutex_lock_interruptible(struct drm_device *dev)
 192 {
 193         struct drm_i915_private *dev_priv = dev->dev_private;
 194         int ret;
 195
 196         ret = i915_gem_check_is_wedged(dev);
 197         if (ret)
 198                 return ret;
 199
 200         ret = mutex_lock_interruptible(&dev->struct_mutex);
 201         if (ret)
 202                 return ret;
 203
 204         if (atomic_read(&dev_priv->mm.wedged)) {
 205                 mutex_unlock(&dev->struct_mutex);
 206                 return -EAGAIN;
 207         }
 208
 209         WARN_ON(i915_verify_lists(dev));
 210         return 0;
 211 }
 212
 213 static inline bool
 214 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
 215 {
 216         return obj_priv->gtt_space &&
 217                 !obj_priv->active &&
 218                 obj_priv->pin_count == 0;
 219 }
 220
 221 int i915_gem_do_init(struct drm_device *dev,
 222                      unsigned long start,
 223                      unsigned long mappable_end,
 224                      unsigned long end)
 225 {
 226         drm_i915_private_t *dev_priv = dev->dev_private;
 227
 228         if (start >= end ||
 229             (start & (PAGE_SIZE - 1)) != 0 ||
 230             (end & (PAGE_SIZE - 1)) != 0) {
 231                 return -EINVAL;
 232         }
 233
 234         drm_mm_init(&dev_priv->mm.gtt_space, start,
 235                     end - start);
 236
 237         dev_priv->mm.gtt_total = end - start;
 238         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 239         dev_priv->mm.gtt_mappable_end = mappable_end;
 240
 241         return 0;
 242 }
 243
 244 int
 245 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 246                     struct drm_file *file_priv)
 247 {
 248         struct drm_i915_gem_init *args = data;
 249         int ret;
 250
 251         mutex_lock(&dev->struct_mutex);
 252         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 253         mutex_unlock(&dev->struct_mutex);
 254
 255         return ret;
 256 }
 257
 258 int
 259 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 260                             struct drm_file *file_priv)
 261 {
 262         struct drm_i915_private *dev_priv = dev->dev_private;
 263         struct drm_i915_gem_get_aperture *args = data;
 264
 265         if (!(dev->driver->driver_features & DRIVER_GEM))
 266                 return -ENODEV;
 267
 268         mutex_lock(&dev->struct_mutex);
 269         args->aper_size = dev_priv->mm.gtt_total;
 270         args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
 271         mutex_unlock(&dev->struct_mutex);
 272
 273         return 0;
 274 }
 275
 276
 277 /**
 278  * Creates a new mm object and returns a handle to it.
 279  */
 280 int
 281 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 282                       struct drm_file *file_priv)
 283 {
 284         struct drm_i915_gem_create *args = data;
 285         struct drm_gem_object *obj;
 286         int ret;
 287         u32 handle;
 288
 289         args->size = roundup(args->size, PAGE_SIZE);
 290
 291         /* Allocate the new object */
 292         obj = i915_gem_alloc_object(dev, args->size);
 293         if (obj == NULL)
 294                 return -ENOMEM;
 295
 296         ret = drm_gem_handle_create(file_priv, obj, &handle);
 297         if (ret) {
 298                 drm_gem_object_release(obj);
 299                 i915_gem_info_remove_obj(dev->dev_private, obj->size);
 300                 kfree(obj);
 301                 return ret;
 302         }
 303
 304         /* drop reference from allocate - handle holds it now */
 305         drm_gem_object_unreference(obj);
 306         trace_i915_gem_object_create(obj);
 307
 308         args->handle = handle;
 309         return 0;
 310 }
 311
 312 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 313 {
 314         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 315         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 316
 317         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 318                 obj_priv->tiling_mode != I915_TILING_NONE;
 319 }
 320
 321 static inline void
 322 slow_shmem_copy(struct page *dst_page,
 323                 int dst_offset,
 324                 struct page *src_page,
 325                 int src_offset,
 326                 int length)
 327 {
 328         char *dst_vaddr, *src_vaddr;
 329
 330         dst_vaddr = kmap(dst_page);
 331         src_vaddr = kmap(src_page);
 332
 333         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 334
 335         kunmap(src_page);
 336         kunmap(dst_page);
 337 }
 338
 339 static inline void
 340 slow_shmem_bit17_copy(struct page *gpu_page,
 341                       int gpu_offset,
 342                       struct page *cpu_page,
 343                       int cpu_offset,
 344                       int length,
 345                       int is_read)
 346 {
 347         char *gpu_vaddr, *cpu_vaddr;
 348
 349         /* Use the unswizzled path if this page isn't affected. */
 350         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 351                 if (is_read)
 352                         return slow_shmem_copy(cpu_page, cpu_offset,
 353                                                gpu_page, gpu_offset, length);
 354                 else
 355                         return slow_shmem_copy(gpu_page, gpu_offset,
 356                                                cpu_page, cpu_offset, length);
 357         }
 358
 359         gpu_vaddr = kmap(gpu_page);
 360         cpu_vaddr = kmap(cpu_page);
 361
 362         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 363          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 364          */
 365         while (length > 0) {
 366                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 367                 int this_length = min(cacheline_end - gpu_offset, length);
 368                 int swizzled_gpu_offset = gpu_offset ^ 64;
 369
 370                 if (is_read) {
 371                         memcpy(cpu_vaddr + cpu_offset,
 372                                gpu_vaddr + swizzled_gpu_offset,
 373                                this_length);
 374                 } else {
 375                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 376                                cpu_vaddr + cpu_offset,
 377                                this_length);
 378                 }
 379                 cpu_offset += this_length;
 380                 gpu_offset += this_length;
 381                 length -= this_length;
 382         }
 383
 384         kunmap(cpu_page);
 385         kunmap(gpu_page);
 386 }
 387
 388 /**
 389  * This is the fast shmem pread path, which attempts to copy_from_user directly
 390  * from the backing pages of the object to the user's address space.  On a
 391  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 392  */
 393 static int
 394 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 395                           struct drm_i915_gem_pread *args,
 396                           struct drm_file *file_priv)
 397 {
 398         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 399         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 400         ssize_t remain;
 401         loff_t offset;
 402         char __user *user_data;
 403         int page_offset, page_length;
 404
 405         user_data = (char __user *) (uintptr_t) args->data_ptr;
 406         remain = args->size;
 407
 408         obj_priv = to_intel_bo(obj);
 409         offset = args->offset;
 410
 411         while (remain > 0) {
 412                 struct page *page;
 413                 char *vaddr;
 414                 int ret;
 415
 416                 /* Operation in this page
 417                  *
 418                  * page_offset = offset within page
 419                  * page_length = bytes to copy for this page
 420                  */
 421                 page_offset = offset & (PAGE_SIZE-1);
 422                 page_length = remain;
 423                 if ((page_offset + remain) > PAGE_SIZE)
 424                         page_length = PAGE_SIZE - page_offset;
 425
 426                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 427                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 428                 if (IS_ERR(page))
 429                         return PTR_ERR(page);
 430
 431                 vaddr = kmap_atomic(page);
 432                 ret = __copy_to_user_inatomic(user_data,
 433                                               vaddr + page_offset,
 434                                               page_length);
 435                 kunmap_atomic(vaddr);
 436
 437                 mark_page_accessed(page);
 438                 page_cache_release(page);
 439                 if (ret)
 440                         return -EFAULT;
 441
 442                 remain -= page_length;
 443                 user_data += page_length;
 444                 offset += page_length;
 445         }
 446
 447         return 0;
 448 }
 449
 450 /**
 451  * This is the fallback shmem pread path, which allocates temporary storage
 452  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 453  * can copy out of the object's backing pages while holding the struct mutex
 454  * and not take page faults.
 455  */
 456 static int
 457 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 458                           struct drm_i915_gem_pread *args,
 459                           struct drm_file *file_priv)
 460 {
 461         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 462         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 463         struct mm_struct *mm = current->mm;
 464         struct page **user_pages;
 465         ssize_t remain;
 466         loff_t offset, pinned_pages, i;
 467         loff_t first_data_page, last_data_page, num_pages;
 468         int shmem_page_offset;
 469         int data_page_index, data_page_offset;
 470         int page_length;
 471         int ret;
 472         uint64_t data_ptr = args->data_ptr;
 473         int do_bit17_swizzling;
 474
 475         remain = args->size;
 476
 477         /* Pin the user pages containing the data.  We can't fault while
 478          * holding the struct mutex, yet we want to hold it while
 479          * dereferencing the user data.
 480          */
 481         first_data_page = data_ptr / PAGE_SIZE;
 482         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 483         num_pages = last_data_page - first_data_page + 1;
 484
 485         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 486         if (user_pages == NULL)
 487                 return -ENOMEM;
 488
 489         mutex_unlock(&dev->struct_mutex);
 490         down_read(&mm->mmap_sem);
 491         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 492                                       num_pages, 1, 0, user_pages, NULL);
 493         up_read(&mm->mmap_sem);
 494         mutex_lock(&dev->struct_mutex);
 495         if (pinned_pages < num_pages) {
 496                 ret = -EFAULT;
 497                 goto out;
 498         }
 499
 500         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 501                                                         args->offset,
 502                                                         args->size);
 503         if (ret)
 504                 goto out;
 505
 506         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 507
 508         obj_priv = to_intel_bo(obj);
 509         offset = args->offset;
 510
 511         while (remain > 0) {
 512                 struct page *page;
 513
 514                 /* Operation in this page
 515                  *
 516                  * shmem_page_offset = offset within page in shmem file
 517                  * data_page_index = page number in get_user_pages return
 518                  * data_page_offset = offset with data_page_index page.
 519                  * page_length = bytes to copy for this page
 520                  */
 521                 shmem_page_offset = offset & ~PAGE_MASK;
 522                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 523                 data_page_offset = data_ptr & ~PAGE_MASK;
 524
 525                 page_length = remain;
 526                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 527                         page_length = PAGE_SIZE - shmem_page_offset;
 528                 if ((data_page_offset + page_length) > PAGE_SIZE)
 529                         page_length = PAGE_SIZE - data_page_offset;
 530
 531                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 532                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 533                 if (IS_ERR(page))
 534                         return PTR_ERR(page);
 535
 536                 if (do_bit17_swizzling) {
 537                         slow_shmem_bit17_copy(page,
 538                                               shmem_page_offset,
 539                                               user_pages[data_page_index],
 540                                               data_page_offset,
 541                                               page_length,
 542                                               1);
 543                 } else {
 544                         slow_shmem_copy(user_pages[data_page_index],
 545                                         data_page_offset,
 546                                         page,
 547                                         shmem_page_offset,
 548                                         page_length);
 549                 }
 550
 551                 mark_page_accessed(page);
 552                 page_cache_release(page);
 553
 554                 remain -= page_length;
 555                 data_ptr += page_length;
 556                 offset += page_length;
 557         }
 558
 559 out:
 560         for (i = 0; i < pinned_pages; i++) {
 561                 SetPageDirty(user_pages[i]);
 562                 mark_page_accessed(user_pages[i]);
 563                 page_cache_release(user_pages[i]);
 564         }
 565         drm_free_large(user_pages);
 566
 567         return ret;
 568 }
 569
 570 /**
 571  * Reads data from the object referenced by handle.
 572  *
 573  * On error, the contents of *data are undefined.
 574  */
 575 int
 576 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 577                      struct drm_file *file_priv)
 578 {
 579         struct drm_i915_gem_pread *args = data;
 580         struct drm_gem_object *obj;
 581         struct drm_i915_gem_object *obj_priv;
 582         int ret = 0;
 583
 584         if (args->size == 0)
 585                 return 0;
 586
 587         if (!access_ok(VERIFY_WRITE,
 588                        (char __user *)(uintptr_t)args->data_ptr,
 589                        args->size))
 590                 return -EFAULT;
 591
 592         ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
 593                                        args->size);
 594         if (ret)
 595                 return -EFAULT;
 596
 597         ret = i915_mutex_lock_interruptible(dev);
 598         if (ret)
 599                 return ret;
 600
 601         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 602         if (obj == NULL) {
 603                 ret = -ENOENT;
 604                 goto unlock;
 605         }
 606         obj_priv = to_intel_bo(obj);
 607
 608         /* Bounds check source.  */
 609         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 610                 ret = -EINVAL;
 611                 goto out;
 612         }
 613
 614         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 615                                                         args->offset,
 616                                                         args->size);
 617         if (ret)
 618                 goto out;
 619
 620         ret = -EFAULT;
 621         if (!i915_gem_object_needs_bit17_swizzle(obj))
 622                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 623         if (ret == -EFAULT)
 624                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 625
 626 out:
 627         drm_gem_object_unreference(obj);
 628 unlock:
 629         mutex_unlock(&dev->struct_mutex);
 630         return ret;
 631 }
 632
 633 /* This is the fast write path which cannot handle
 634  * page faults in the source data
 635  */
 636
 637 static inline int
 638 fast_user_write(struct io_mapping *mapping,
 639                 loff_t page_base, int page_offset,
 640                 char __user *user_data,
 641                 int length)
 642 {
 643         char *vaddr_atomic;
 644         unsigned long unwritten;
 645
 646         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 647         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 648                                                       user_data, length);
 649         io_mapping_unmap_atomic(vaddr_atomic);
 650         return unwritten;
 651 }
 652
 653 /* Here's the write path which can sleep for
 654  * page faults
 655  */
 656
 657 static inline void
 658 slow_kernel_write(struct io_mapping *mapping,
 659                   loff_t gtt_base, int gtt_offset,
 660                   struct page *user_page, int user_offset,
 661                   int length)
 662 {
 663         char __iomem *dst_vaddr;
 664         char *src_vaddr;
 665
 666         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 667         src_vaddr = kmap(user_page);
 668
 669         memcpy_toio(dst_vaddr + gtt_offset,
 670                     src_vaddr + user_offset,
 671                     length);
 672
 673         kunmap(user_page);
 674         io_mapping_unmap(dst_vaddr);
 675 }
 676
 677 /**
 678  * This is the fast pwrite path, where we copy the data directly from the
 679  * user into the GTT, uncached.
 680  */
 681 static int
 682 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 683                          struct drm_i915_gem_pwrite *args,
 684                          struct drm_file *file_priv)
 685 {
 686         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 687         drm_i915_private_t *dev_priv = dev->dev_private;
 688         ssize_t remain;
 689         loff_t offset, page_base;
 690         char __user *user_data;
 691         int page_offset, page_length;
 692
 693         user_data = (char __user *) (uintptr_t) args->data_ptr;
 694         remain = args->size;
 695
 696         obj_priv = to_intel_bo(obj);
 697         offset = obj_priv->gtt_offset + args->offset;
 698
 699         while (remain > 0) {
 700                 /* Operation in this page
 701                  *
 702                  * page_base = page offset within aperture
 703                  * page_offset = offset within page
 704                  * page_length = bytes to copy for this page
 705                  */
 706                 page_base = (offset & ~(PAGE_SIZE-1));
 707                 page_offset = offset & (PAGE_SIZE-1);
 708                 page_length = remain;
 709                 if ((page_offset + remain) > PAGE_SIZE)
 710                         page_length = PAGE_SIZE - page_offset;
 711
 712                 /* If we get a fault while copying data, then (presumably) our
 713                  * source page isn't available.  Return the error and we'll
 714                  * retry in the slow path.
 715                  */
 716                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 717                                     page_offset, user_data, page_length))
 718
 719                         return -EFAULT;
 720
 721                 remain -= page_length;
 722                 user_data += page_length;
 723                 offset += page_length;
 724         }
 725
 726         return 0;
 727 }
 728
 729 /**
 730  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 731  * the memory and maps it using kmap_atomic for copying.
 732  *
 733  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 734  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 735  */
 736 static int
 737 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 738                          struct drm_i915_gem_pwrite *args,
 739                          struct drm_file *file_priv)
 740 {
 741         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 742         drm_i915_private_t *dev_priv = dev->dev_private;
 743         ssize_t remain;
 744         loff_t gtt_page_base, offset;
 745         loff_t first_data_page, last_data_page, num_pages;
 746         loff_t pinned_pages, i;
 747         struct page **user_pages;
 748         struct mm_struct *mm = current->mm;
 749         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 750         int ret;
 751         uint64_t data_ptr = args->data_ptr;
 752
 753         remain = args->size;
 754
 755         /* Pin the user pages containing the data.  We can't fault while
 756          * holding the struct mutex, and all of the pwrite implementations
 757          * want to hold it while dereferencing the user data.
 758          */
 759         first_data_page = data_ptr / PAGE_SIZE;
 760         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 761         num_pages = last_data_page - first_data_page + 1;
 762
 763         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 764         if (user_pages == NULL)
 765                 return -ENOMEM;
 766
 767         mutex_unlock(&dev->struct_mutex);
 768         down_read(&mm->mmap_sem);
 769         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 770                                       num_pages, 0, 0, user_pages, NULL);
 771         up_read(&mm->mmap_sem);
 772         mutex_lock(&dev->struct_mutex);
 773         if (pinned_pages < num_pages) {
 774                 ret = -EFAULT;
 775                 goto out_unpin_pages;
 776         }
 777
 778         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 779         if (ret)
 780                 goto out_unpin_pages;
 781
 782         obj_priv = to_intel_bo(obj);
 783         offset = obj_priv->gtt_offset + args->offset;
 784
 785         while (remain > 0) {
 786                 /* Operation in this page
 787                  *
 788                  * gtt_page_base = page offset within aperture
 789                  * gtt_page_offset = offset within page in aperture
 790                  * data_page_index = page number in get_user_pages return
 791                  * data_page_offset = offset with data_page_index page.
 792                  * page_length = bytes to copy for this page
 793                  */
 794                 gtt_page_base = offset & PAGE_MASK;
 795                 gtt_page_offset = offset & ~PAGE_MASK;
 796                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 797                 data_page_offset = data_ptr & ~PAGE_MASK;
 798
 799                 page_length = remain;
 800                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 801                         page_length = PAGE_SIZE - gtt_page_offset;
 802                 if ((data_page_offset + page_length) > PAGE_SIZE)
 803                         page_length = PAGE_SIZE - data_page_offset;
 804
 805                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 806                                   gtt_page_base, gtt_page_offset,
 807                                   user_pages[data_page_index],
 808                                   data_page_offset,
 809                                   page_length);
 810
 811                 remain -= page_length;
 812                 offset += page_length;
 813                 data_ptr += page_length;
 814         }
 815
 816 out_unpin_pages:
 817         for (i = 0; i < pinned_pages; i++)
 818                 page_cache_release(user_pages[i]);
 819         drm_free_large(user_pages);
 820
 821         return ret;
 822 }
 823
 824 /**
 825  * This is the fast shmem pwrite path, which attempts to directly
 826  * copy_from_user into the kmapped pages backing the object.
 827  */
 828 static int
 829 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 830                            struct drm_i915_gem_pwrite *args,
 831                            struct drm_file *file_priv)
 832 {
 833         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 834         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 835         ssize_t remain;
 836         loff_t offset;
 837         char __user *user_data;
 838         int page_offset, page_length;
 839
 840         user_data = (char __user *) (uintptr_t) args->data_ptr;
 841         remain = args->size;
 842
 843         obj_priv = to_intel_bo(obj);
 844         offset = args->offset;
 845         obj_priv->dirty = 1;
 846
 847         while (remain > 0) {
 848                 struct page *page;
 849                 char *vaddr;
 850                 int ret;
 851
 852                 /* Operation in this page
 853                  *
 854                  * page_offset = offset within page
 855                  * page_length = bytes to copy for this page
 856                  */
 857                 page_offset = offset & (PAGE_SIZE-1);
 858                 page_length = remain;
 859                 if ((page_offset + remain) > PAGE_SIZE)
 860                         page_length = PAGE_SIZE - page_offset;
 861
 862                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 863                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 864                 if (IS_ERR(page))
 865                         return PTR_ERR(page);
 866
 867                 vaddr = kmap_atomic(page, KM_USER0);
 868                 ret = __copy_from_user_inatomic(vaddr + page_offset,
 869                                                 user_data,
 870                                                 page_length);
 871                 kunmap_atomic(vaddr, KM_USER0);
 872
 873                 set_page_dirty(page);
 874                 mark_page_accessed(page);
 875                 page_cache_release(page);
 876
 877                 /* If we get a fault while copying data, then (presumably) our
 878                  * source page isn't available.  Return the error and we'll
 879                  * retry in the slow path.
 880                  */
 881                 if (ret)
 882                         return -EFAULT;
 883
 884                 remain -= page_length;
 885                 user_data += page_length;
 886                 offset += page_length;
 887         }
 888
 889         return 0;
 890 }
 891
 892 /**
 893  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 894  * the memory and maps it using kmap_atomic for copying.
 895  *
 896  * This avoids taking mmap_sem for faulting on the user's address while the
 897  * struct_mutex is held.
 898  */
 899 static int
 900 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 901                            struct drm_i915_gem_pwrite *args,
 902                            struct drm_file *file_priv)
 903 {
 904         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
 905         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 906         struct mm_struct *mm = current->mm;
 907         struct page **user_pages;
 908         ssize_t remain;
 909         loff_t offset, pinned_pages, i;
 910         loff_t first_data_page, last_data_page, num_pages;
 911         int shmem_page_offset;
 912         int data_page_index,  data_page_offset;
 913         int page_length;
 914         int ret;
 915         uint64_t data_ptr = args->data_ptr;
 916         int do_bit17_swizzling;
 917
 918         remain = args->size;
 919
 920         /* Pin the user pages containing the data.  We can't fault while
 921          * holding the struct mutex, and all of the pwrite implementations
 922          * want to hold it while dereferencing the user data.
 923          */
 924         first_data_page = data_ptr / PAGE_SIZE;
 925         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 926         num_pages = last_data_page - first_data_page + 1;
 927
 928         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 929         if (user_pages == NULL)
 930                 return -ENOMEM;
 931
 932         mutex_unlock(&dev->struct_mutex);
 933         down_read(&mm->mmap_sem);
 934         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 935                                       num_pages, 0, 0, user_pages, NULL);
 936         up_read(&mm->mmap_sem);
 937         mutex_lock(&dev->struct_mutex);
 938         if (pinned_pages < num_pages) {
 939                 ret = -EFAULT;
 940                 goto out;
 941         }
 942
 943         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 944         if (ret)
 945                 goto out;
 946
 947         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 948
 949         obj_priv = to_intel_bo(obj);
 950         offset = args->offset;
 951         obj_priv->dirty = 1;
 952
 953         while (remain > 0) {
 954                 struct page *page;
 955
 956                 /* Operation in this page
 957                  *
 958                  * shmem_page_offset = offset within page in shmem file
 959                  * data_page_index = page number in get_user_pages return
 960                  * data_page_offset = offset with data_page_index page.
 961                  * page_length = bytes to copy for this page
 962                  */
 963                 shmem_page_offset = offset & ~PAGE_MASK;
 964                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 965                 data_page_offset = data_ptr & ~PAGE_MASK;
 966
 967                 page_length = remain;
 968                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 969                         page_length = PAGE_SIZE - shmem_page_offset;
 970                 if ((data_page_offset + page_length) > PAGE_SIZE)
 971                         page_length = PAGE_SIZE - data_page_offset;
 972
 973                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 974                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 975                 if (IS_ERR(page)) {
 976                         ret = PTR_ERR(page);
 977                         goto out;
 978                 }
 979
 980                 if (do_bit17_swizzling) {
 981                         slow_shmem_bit17_copy(page,
 982                                               shmem_page_offset,
 983                                               user_pages[data_page_index],
 984                                               data_page_offset,
 985                                               page_length,
 986                                               0);
 987                 } else {
 988                         slow_shmem_copy(page,
 989                                         shmem_page_offset,
 990                                         user_pages[data_page_index],
 991                                         data_page_offset,
 992                                         page_length);
 993                 }
 994
 995                 set_page_dirty(page);
 996                 mark_page_accessed(page);
 997                 page_cache_release(page);
 998
 999                 remain -= page_length;
1000                 data_ptr += page_length;
1001                 offset += page_length;
1002         }
1003
1004 out:
1005         for (i = 0; i < pinned_pages; i++)
1006                 page_cache_release(user_pages[i]);
1007         drm_free_large(user_pages);
1008
1009         return ret;
1010 }
1011
1012 /**
1013  * Writes data to the object referenced by handle.
1014  *
1015  * On error, the contents of the buffer that were to be modified are undefined.
1016  */
1017 int
1018 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1019                       struct drm_file *file)
1020 {
1021         struct drm_i915_gem_pwrite *args = data;
1022         struct drm_gem_object *obj;
1023         struct drm_i915_gem_object *obj_priv;
1024         int ret;
1025
1026         if (args->size == 0)
1027                 return 0;
1028
1029         if (!access_ok(VERIFY_READ,
1030                        (char __user *)(uintptr_t)args->data_ptr,
1031                        args->size))
1032                 return -EFAULT;
1033
1034         ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1035                                       args->size);
1036         if (ret)
1037                 return -EFAULT;
1038
1039         ret = i915_mutex_lock_interruptible(dev);
1040         if (ret)
1041                 return ret;
1042
1043         obj = drm_gem_object_lookup(dev, file, args->handle);
1044         if (obj == NULL) {
1045                 ret = -ENOENT;
1046                 goto unlock;
1047         }
1048         obj_priv = to_intel_bo(obj);
1049
1050         /* Bounds check destination. */
1051         if (args->offset > obj->size || args->size > obj->size - args->offset) {
1052                 ret = -EINVAL;
1053                 goto out;
1054         }
1055
1056         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1057          * it would end up going through the fenced access, and we'll get
1058          * different detiling behavior between reading and writing.
1059          * pread/pwrite currently are reading and writing from the CPU
1060          * perspective, requiring manual detiling by the client.
1061          */
1062         if (obj_priv->phys_obj)
1063                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1064         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
1065                  obj_priv->gtt_space &&
1066                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
1067                 ret = i915_gem_object_pin(obj, 0, true);
1068                 if (ret)
1069                         goto out;
1070
1071                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1072                 if (ret)
1073                         goto out_unpin;
1074
1075                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1076                 if (ret == -EFAULT)
1077                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1078
1079 out_unpin:
1080                 i915_gem_object_unpin(obj);
1081         } else {
1082                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1083                 if (ret)
1084                         goto out;
1085
1086                 ret = -EFAULT;
1087                 if (!i915_gem_object_needs_bit17_swizzle(obj))
1088                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1089                 if (ret == -EFAULT)
1090                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1091         }
1092
1093 out:
1094         drm_gem_object_unreference(obj);
1095 unlock:
1096         mutex_unlock(&dev->struct_mutex);
1097         return ret;
1098 }
1099
1100 /**
1101  * Called when user space prepares to use an object with the CPU, either
1102  * through the mmap ioctl's mapping or a GTT mapping.
1103  */
1104 int
1105 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1106                           struct drm_file *file_priv)
1107 {
1108         struct drm_i915_private *dev_priv = dev->dev_private;
1109         struct drm_i915_gem_set_domain *args = data;
1110         struct drm_gem_object *obj;
1111         struct drm_i915_gem_object *obj_priv;
1112         uint32_t read_domains = args->read_domains;
1113         uint32_t write_domain = args->write_domain;
1114         int ret;
1115
1116         if (!(dev->driver->driver_features & DRIVER_GEM))
1117                 return -ENODEV;
1118
1119         /* Only handle setting domains to types used by the CPU. */
1120         if (write_domain & I915_GEM_GPU_DOMAINS)
1121                 return -EINVAL;
1122
1123         if (read_domains & I915_GEM_GPU_DOMAINS)
1124                 return -EINVAL;
1125
1126         /* Having something in the write domain implies it's in the read
1127          * domain, and only that read domain.  Enforce that in the request.
1128          */
1129         if (write_domain != 0 && read_domains != write_domain)
1130                 return -EINVAL;
1131
1132         ret = i915_mutex_lock_interruptible(dev);
1133         if (ret)
1134                 return ret;
1135
1136         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1137         if (obj == NULL) {
1138                 ret = -ENOENT;
1139                 goto unlock;
1140         }
1141         obj_priv = to_intel_bo(obj);
1142
1143         intel_mark_busy(dev, obj);
1144
1145         if (read_domains & I915_GEM_DOMAIN_GTT) {
1146                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1147
1148                 /* Update the LRU on the fence for the CPU access that's
1149                  * about to occur.
1150                  */
1151                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1152                         struct drm_i915_fence_reg *reg =
1153                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1154                         list_move_tail(&reg->lru_list,
1155                                        &dev_priv->mm.fence_list);
1156                 }
1157
1158                 /* Silently promote "you're not bound, there was nothing to do"
1159                  * to success, since the client was just asking us to
1160                  * make sure everything was done.
1161                  */
1162                 if (ret == -EINVAL)
1163                         ret = 0;
1164         } else {
1165                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1166         }
1167
1168         /* Maintain LRU order of "inactive" objects */
1169         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1170                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1171
1172         drm_gem_object_unreference(obj);
1173 unlock:
1174         mutex_unlock(&dev->struct_mutex);
1175         return ret;
1176 }
1177
1178 /**
1179  * Called when user space has done writes to this buffer
1180  */
1181 int
1182 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1183                       struct drm_file *file_priv)
1184 {
1185         struct drm_i915_gem_sw_finish *args = data;
1186         struct drm_gem_object *obj;
1187         int ret = 0;
1188
1189         if (!(dev->driver->driver_features & DRIVER_GEM))
1190                 return -ENODEV;
1191
1192         ret = i915_mutex_lock_interruptible(dev);
1193         if (ret)
1194                 return ret;
1195
1196         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1197         if (obj == NULL) {
1198                 ret = -ENOENT;
1199                 goto unlock;
1200         }
1201
1202         /* Pinned buffers may be scanout, so flush the cache */
1203         if (to_intel_bo(obj)->pin_count)
1204                 i915_gem_object_flush_cpu_write_domain(obj);
1205
1206         drm_gem_object_unreference(obj);
1207 unlock:
1208         mutex_unlock(&dev->struct_mutex);
1209         return ret;
1210 }
1211
1212 /**
1213  * Maps the contents of an object, returning the address it is mapped
1214  * into.
1215  *
1216  * While the mapping holds a reference on the contents of the object, it doesn't
1217  * imply a ref on the object itself.
1218  */
1219 int
1220 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1221                    struct drm_file *file_priv)
1222 {
1223         struct drm_i915_private *dev_priv = dev->dev_private;
1224         struct drm_i915_gem_mmap *args = data;
1225         struct drm_gem_object *obj;
1226         loff_t offset;
1227         unsigned long addr;
1228
1229         if (!(dev->driver->driver_features & DRIVER_GEM))
1230                 return -ENODEV;
1231
1232         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1233         if (obj == NULL)
1234                 return -ENOENT;
1235
1236         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1237                 drm_gem_object_unreference_unlocked(obj);
1238                 return -E2BIG;
1239         }
1240
1241         offset = args->offset;
1242
1243         down_write(&current->mm->mmap_sem);
1244         addr = do_mmap(obj->filp, 0, args->size,
1245                        PROT_READ | PROT_WRITE, MAP_SHARED,
1246                        args->offset);
1247         up_write(&current->mm->mmap_sem);
1248         drm_gem_object_unreference_unlocked(obj);
1249         if (IS_ERR((void *)addr))
1250                 return addr;
1251
1252         args->addr_ptr = (uint64_t) addr;
1253
1254         return 0;
1255 }
1256
1257 /**
1258  * i915_gem_fault - fault a page into the GTT
1259  * vma: VMA in question
1260  * vmf: fault info
1261  *
1262  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1263  * from userspace.  The fault handler takes care of binding the object to
1264  * the GTT (if needed), allocating and programming a fence register (again,
1265  * only if needed based on whether the old reg is still valid or the object
1266  * is tiled) and inserting a new PTE into the faulting process.
1267  *
1268  * Note that the faulting process may involve evicting existing objects
1269  * from the GTT and/or fence registers to make room.  So performance may
1270  * suffer if the GTT working set is large or there are few fence registers
1271  * left.
1272  */
1273 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1274 {
1275         struct drm_gem_object *obj = vma->vm_private_data;
1276         struct drm_device *dev = obj->dev;
1277         drm_i915_private_t *dev_priv = dev->dev_private;
1278         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1279         pgoff_t page_offset;
1280         unsigned long pfn;
1281         int ret = 0;
1282         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1283
1284         /* We don't use vmf->pgoff since that has the fake offset */
1285         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1286                 PAGE_SHIFT;
1287
1288         /* Now bind it into the GTT if needed */
1289         mutex_lock(&dev->struct_mutex);
1290         BUG_ON(obj_priv->pin_count && !obj_priv->pin_mappable);
1291
1292         if (obj_priv->gtt_space) {
1293                 if (!obj_priv->map_and_fenceable) {
1294                         ret = i915_gem_object_unbind(obj);
1295                         if (ret)
1296                                 goto unlock;
1297                 }
1298         }
1299
1300         if (!obj_priv->gtt_space) {
1301                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1302                 if (ret)
1303                         goto unlock;
1304         }
1305
1306         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1307         if (ret)
1308                 goto unlock;
1309
1310         if (!obj_priv->fault_mappable) {
1311                 obj_priv->fault_mappable = true;
1312                 i915_gem_info_update_mappable(dev_priv, obj_priv, true);
1313         }
1314
1315         /* Need a new fence register? */
1316         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1317                 ret = i915_gem_object_get_fence_reg(obj, true);
1318                 if (ret)
1319                         goto unlock;
1320         }
1321
1322         if (i915_gem_object_is_inactive(obj_priv))
1323                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1324
1325         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1326                 page_offset;
1327
1328         /* Finally, remap it using the new GTT offset */
1329         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1330 unlock:
1331         mutex_unlock(&dev->struct_mutex);
1332
1333         switch (ret) {
1334         case -EAGAIN:
1335                 set_need_resched();
1336         case 0:
1337         case -ERESTARTSYS:
1338                 return VM_FAULT_NOPAGE;
1339         case -ENOMEM:
1340                 return VM_FAULT_OOM;
1341         default:
1342                 return VM_FAULT_SIGBUS;
1343         }
1344 }
1345
1346 /**
1347  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1348  * @obj: obj in question
1349  *
1350  * GEM memory mapping works by handing back to userspace a fake mmap offset
1351  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1352  * up the object based on the offset and sets up the various memory mapping
1353  * structures.
1354  *
1355  * This routine allocates and attaches a fake offset for @obj.
1356  */
1357 static int
1358 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1359 {
1360         struct drm_device *dev = obj->dev;
1361         struct drm_gem_mm *mm = dev->mm_private;
1362         struct drm_map_list *list;
1363         struct drm_local_map *map;
1364         int ret = 0;
1365
1366         /* Set the object up for mmap'ing */
1367         list = &obj->map_list;
1368         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1369         if (!list->map)
1370                 return -ENOMEM;
1371
1372         map = list->map;
1373         map->type = _DRM_GEM;
1374         map->size = obj->size;
1375         map->handle = obj;
1376
1377         /* Get a DRM GEM mmap offset allocated... */
1378         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1379                                                     obj->size / PAGE_SIZE, 0, 0);
1380         if (!list->file_offset_node) {
1381                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1382                 ret = -ENOSPC;
1383                 goto out_free_list;
1384         }
1385
1386         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1387                                                   obj->size / PAGE_SIZE, 0);
1388         if (!list->file_offset_node) {
1389                 ret = -ENOMEM;
1390                 goto out_free_list;
1391         }
1392
1393         list->hash.key = list->file_offset_node->start;
1394         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1395         if (ret) {
1396                 DRM_ERROR("failed to add to map hash\n");
1397                 goto out_free_mm;
1398         }
1399
1400         return 0;
1401
1402 out_free_mm:
1403         drm_mm_put_block(list->file_offset_node);
1404 out_free_list:
1405         kfree(list->map);
1406         list->map = NULL;
1407
1408         return ret;
1409 }
1410
1411 /**
1412  * i915_gem_release_mmap - remove physical page mappings
1413  * @obj: obj in question
1414  *
1415  * Preserve the reservation of the mmapping with the DRM core code, but
1416  * relinquish ownership of the pages back to the system.
1417  *
1418  * It is vital that we remove the page mapping if we have mapped a tiled
1419  * object through the GTT and then lose the fence register due to
1420  * resource pressure. Similarly if the object has been moved out of the
1421  * aperture, than pages mapped into userspace must be revoked. Removing the
1422  * mapping will then trigger a page fault on the next user access, allowing
1423  * fixup by i915_gem_fault().
1424  */
1425 void
1426 i915_gem_release_mmap(struct drm_gem_object *obj)
1427 {
1428         struct drm_device *dev = obj->dev;
1429         struct drm_i915_private *dev_priv = dev->dev_private;
1430         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1431
1432         if (unlikely(obj->map_list.map && dev->dev_mapping))
1433                 unmap_mapping_range(dev->dev_mapping,
1434                                     (loff_t)obj->map_list.hash.key<<PAGE_SHIFT,
1435                                     obj->size, 1);
1436
1437         if (obj_priv->fault_mappable) {
1438                 obj_priv->fault_mappable = false;
1439                 i915_gem_info_update_mappable(dev_priv, obj_priv, false);
1440         }
1441 }
1442
1443 static void
1444 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1445 {
1446         struct drm_device *dev = obj->dev;
1447         struct drm_gem_mm *mm = dev->mm_private;
1448         struct drm_map_list *list = &obj->map_list;
1449
1450         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1451         drm_mm_put_block(list->file_offset_node);
1452         kfree(list->map);
1453         list->map = NULL;
1454 }
1455
1456 /**
1457  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1458  * @obj: object to check
1459  *
1460  * Return the required GTT alignment for an object, taking into account
1461  * potential fence register mapping.
1462  */
1463 static uint32_t
1464 i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj_priv)
1465 {
1466         struct drm_device *dev = obj_priv->base.dev;
1467
1468         /*
1469          * Minimum alignment is 4k (GTT page size), but might be greater
1470          * if a fence register is needed for the object.
1471          */
1472         if (INTEL_INFO(dev)->gen >= 4 ||
1473             obj_priv->tiling_mode == I915_TILING_NONE)
1474                 return 4096;
1475
1476         /*
1477          * Previous chips need to be aligned to the size of the smallest
1478          * fence register that can contain the object.
1479          */
1480         return i915_gem_get_gtt_size(obj_priv);
1481 }
1482
1483 /**
1484  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1485  *                                       unfenced object
1486  * @obj: object to check
1487  *
1488  * Return the required GTT alignment for an object, only taking into account
1489  * unfenced tiled surface requirements.
1490  */
1491 static uint32_t
1492 i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj_priv)
1493 {
1494         struct drm_device *dev = obj_priv->base.dev;
1495         int tile_height;
1496
1497         /*
1498          * Minimum alignment is 4k (GTT page size) for sane hw.
1499          */
1500         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1501             obj_priv->tiling_mode == I915_TILING_NONE)
1502                 return 4096;
1503
1504         /*
1505          * Older chips need unfenced tiled buffers to be aligned to the left
1506          * edge of an even tile row (where tile rows are counted as if the bo is
1507          * placed in a fenced gtt region).
1508          */
1509         if (IS_GEN2(dev) ||
1510             (obj_priv->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
1511                 tile_height = 32;
1512         else
1513                 tile_height = 8;
1514
1515         return tile_height * obj_priv->stride * 2;
1516 }
1517
1518 static uint32_t
1519 i915_gem_get_gtt_size(struct drm_i915_gem_object *obj_priv)
1520 {
1521         struct drm_device *dev = obj_priv->base.dev;
1522         uint32_t size;
1523
1524         /*
1525          * Minimum alignment is 4k (GTT page size), but might be greater
1526          * if a fence register is needed for the object.
1527          */
1528         if (INTEL_INFO(dev)->gen >= 4)
1529                 return obj_priv->base.size;
1530
1531         /*
1532          * Previous chips need to be aligned to the size of the smallest
1533          * fence register that can contain the object.
1534          */
1535         if (INTEL_INFO(dev)->gen == 3)
1536                 size = 1024*1024;
1537         else
1538                 size = 512*1024;
1539
1540         while (size < obj_priv->base.size)
1541                 size <<= 1;
1542
1543         return size;
1544 }
1545
1546 /**
1547  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1548  * @dev: DRM device
1549  * @data: GTT mapping ioctl data
1550  * @file_priv: GEM object info
1551  *
1552  * Simply returns the fake offset to userspace so it can mmap it.
1553  * The mmap call will end up in drm_gem_mmap(), which will set things
1554  * up so we can get faults in the handler above.
1555  *
1556  * The fault handler will take care of binding the object into the GTT
1557  * (since it may have been evicted to make room for something), allocating
1558  * a fence register, and mapping the appropriate aperture address into
1559  * userspace.
1560  */
1561 int
1562 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1563                         struct drm_file *file_priv)
1564 {
1565         struct drm_i915_private *dev_priv = dev->dev_private;
1566         struct drm_i915_gem_mmap_gtt *args = data;
1567         struct drm_gem_object *obj;
1568         struct drm_i915_gem_object *obj_priv;
1569         int ret;
1570
1571         if (!(dev->driver->driver_features & DRIVER_GEM))
1572                 return -ENODEV;
1573
1574         ret = i915_mutex_lock_interruptible(dev);
1575         if (ret)
1576                 return ret;
1577
1578         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1579         if (obj == NULL) {
1580                 ret = -ENOENT;
1581                 goto unlock;
1582         }
1583         obj_priv = to_intel_bo(obj);
1584
1585         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1586                 ret = -E2BIG;
1587                 goto unlock;
1588         }
1589
1590         if (obj_priv->madv != I915_MADV_WILLNEED) {
1591                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1592                 ret = -EINVAL;
1593                 goto out;
1594         }
1595
1596         if (!obj->map_list.map) {
1597                 ret = i915_gem_create_mmap_offset(obj);
1598                 if (ret)
1599                         goto out;
1600         }
1601
1602         args->offset = (u64)obj->map_list.hash.key << PAGE_SHIFT;
1603
1604 out:
1605         drm_gem_object_unreference(obj);
1606 unlock:
1607         mutex_unlock(&dev->struct_mutex);
1608         return ret;
1609 }
1610
1611 static int
1612 i915_gem_object_get_pages_gtt(struct drm_gem_object *obj,
1613                               gfp_t gfpmask)
1614 {
1615         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1616         int page_count, i;
1617         struct address_space *mapping;
1618         struct inode *inode;
1619         struct page *page;
1620
1621         /* Get the list of pages out of our struct file.  They'll be pinned
1622          * at this point until we release them.
1623          */
1624         page_count = obj->size / PAGE_SIZE;
1625         BUG_ON(obj_priv->pages != NULL);
1626         obj_priv->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1627         if (obj_priv->pages == NULL)
1628                 return -ENOMEM;
1629
1630         inode = obj->filp->f_path.dentry->d_inode;
1631         mapping = inode->i_mapping;
1632         for (i = 0; i < page_count; i++) {
1633                 page = read_cache_page_gfp(mapping, i,
1634                                            GFP_HIGHUSER |
1635                                            __GFP_COLD |
1636                                            __GFP_RECLAIMABLE |
1637                                            gfpmask);
1638                 if (IS_ERR(page))
1639                         goto err_pages;
1640
1641                 obj_priv->pages[i] = page;
1642         }
1643
1644         if (obj_priv->tiling_mode != I915_TILING_NONE)
1645                 i915_gem_object_do_bit_17_swizzle(obj);
1646
1647         return 0;
1648
1649 err_pages:
1650         while (i--)
1651                 page_cache_release(obj_priv->pages[i]);
1652
1653         drm_free_large(obj_priv->pages);
1654         obj_priv->pages = NULL;
1655         return PTR_ERR(page);
1656 }
1657
1658 static void
1659 i915_gem_object_put_pages_gtt(struct drm_gem_object *obj)
1660 {
1661         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1662         int page_count = obj->size / PAGE_SIZE;
1663         int i;
1664
1665         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1666
1667         if (obj_priv->tiling_mode != I915_TILING_NONE)
1668                 i915_gem_object_save_bit_17_swizzle(obj);
1669
1670         if (obj_priv->madv == I915_MADV_DONTNEED)
1671                 obj_priv->dirty = 0;
1672
1673         for (i = 0; i < page_count; i++) {
1674                 if (obj_priv->dirty)
1675                         set_page_dirty(obj_priv->pages[i]);
1676
1677                 if (obj_priv->madv == I915_MADV_WILLNEED)
1678                         mark_page_accessed(obj_priv->pages[i]);
1679
1680                 page_cache_release(obj_priv->pages[i]);
1681         }
1682         obj_priv->dirty = 0;
1683
1684         drm_free_large(obj_priv->pages);
1685         obj_priv->pages = NULL;
1686 }
1687
1688 static uint32_t
1689 i915_gem_next_request_seqno(struct drm_device *dev,
1690                             struct intel_ring_buffer *ring)
1691 {
1692         drm_i915_private_t *dev_priv = dev->dev_private;
1693         return ring->outstanding_lazy_request = dev_priv->next_seqno;
1694 }
1695
1696 static void
1697 i915_gem_object_move_to_active(struct drm_gem_object *obj,
1698                                struct intel_ring_buffer *ring)
1699 {
1700         struct drm_device *dev = obj->dev;
1701         struct drm_i915_private *dev_priv = dev->dev_private;
1702         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1703         uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1704
1705         BUG_ON(ring == NULL);
1706         obj_priv->ring = ring;
1707
1708         /* Add a reference if we're newly entering the active list. */
1709         if (!obj_priv->active) {
1710                 drm_gem_object_reference(obj);
1711                 obj_priv->active = 1;
1712         }
1713
1714         /* Move from whatever list we were on to the tail of execution. */
1715         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list);
1716         list_move_tail(&obj_priv->ring_list, &ring->active_list);
1717         obj_priv->last_rendering_seqno = seqno;
1718 }
1719
1720 static void
1721 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1722 {
1723         struct drm_device *dev = obj->dev;
1724         drm_i915_private_t *dev_priv = dev->dev_private;
1725         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1726
1727         BUG_ON(!obj_priv->active);
1728         list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list);
1729         list_del_init(&obj_priv->ring_list);
1730         obj_priv->last_rendering_seqno = 0;
1731 }
1732
1733 /* Immediately discard the backing storage */
1734 static void
1735 i915_gem_object_truncate(struct drm_gem_object *obj)
1736 {
1737         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1738         struct inode *inode;
1739
1740         /* Our goal here is to return as much of the memory as
1741          * is possible back to the system as we are called from OOM.
1742          * To do this we must instruct the shmfs to drop all of its
1743          * backing pages, *now*. Here we mirror the actions taken
1744          * when by shmem_delete_inode() to release the backing store.
1745          */
1746         inode = obj->filp->f_path.dentry->d_inode;
1747         truncate_inode_pages(inode->i_mapping, 0);
1748         if (inode->i_op->truncate_range)
1749                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1750
1751         obj_priv->madv = __I915_MADV_PURGED;
1752 }
1753
1754 static inline int
1755 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1756 {
1757         return obj_priv->madv == I915_MADV_DONTNEED;
1758 }
1759
1760 static void
1761 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1762 {
1763         struct drm_device *dev = obj->dev;
1764         drm_i915_private_t *dev_priv = dev->dev_private;
1765         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1766
1767         if (obj_priv->pin_count != 0)
1768                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list);
1769         else
1770                 list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
1771         list_del_init(&obj_priv->ring_list);
1772
1773         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1774
1775         obj_priv->last_rendering_seqno = 0;
1776         obj_priv->ring = NULL;
1777         if (obj_priv->active) {
1778                 obj_priv->active = 0;
1779                 drm_gem_object_unreference(obj);
1780         }
1781         WARN_ON(i915_verify_lists(dev));
1782 }
1783
1784 static void
1785 i915_gem_process_flushing_list(struct drm_device *dev,
1786                                uint32_t flush_domains,
1787                                struct intel_ring_buffer *ring)
1788 {
1789         drm_i915_private_t *dev_priv = dev->dev_private;
1790         struct drm_i915_gem_object *obj_priv, *next;
1791
1792         list_for_each_entry_safe(obj_priv, next,
1793                                  &ring->gpu_write_list,
1794                                  gpu_write_list) {
1795                 struct drm_gem_object *obj = &obj_priv->base;
1796
1797                 if (obj->write_domain & flush_domains) {
1798                         uint32_t old_write_domain = obj->write_domain;
1799
1800                         obj->write_domain = 0;
1801                         list_del_init(&obj_priv->gpu_write_list);
1802                         i915_gem_object_move_to_active(obj, ring);
1803
1804                         /* update the fence lru list */
1805                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1806                                 struct drm_i915_fence_reg *reg =
1807                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1808                                 list_move_tail(&reg->lru_list,
1809                                                 &dev_priv->mm.fence_list);
1810                         }
1811
1812                         trace_i915_gem_object_change_domain(obj,
1813                                                             obj->read_domains,
1814                                                             old_write_domain);
1815                 }
1816         }
1817 }
1818
1819 int
1820 i915_add_request(struct drm_device *dev,
1821                  struct drm_file *file,
1822                  struct drm_i915_gem_request *request,
1823                  struct intel_ring_buffer *ring)
1824 {
1825         drm_i915_private_t *dev_priv = dev->dev_private;
1826         struct drm_i915_file_private *file_priv = NULL;
1827         uint32_t seqno;
1828         int was_empty;
1829         int ret;
1830
1831         BUG_ON(request == NULL);
1832
1833         if (file != NULL)
1834                 file_priv = file->driver_priv;
1835
1836         ret = ring->add_request(ring, &seqno);
1837         if (ret)
1838             return ret;
1839
1840         ring->outstanding_lazy_request = false;
1841
1842         request->seqno = seqno;
1843         request->ring = ring;
1844         request->emitted_jiffies = jiffies;
1845         was_empty = list_empty(&ring->request_list);
1846         list_add_tail(&request->list, &ring->request_list);
1847
1848         if (file_priv) {
1849                 spin_lock(&file_priv->mm.lock);
1850                 request->file_priv = file_priv;
1851                 list_add_tail(&request->client_list,
1852                               &file_priv->mm.request_list);
1853                 spin_unlock(&file_priv->mm.lock);
1854         }
1855
1856         if (!dev_priv->mm.suspended) {
1857                 mod_timer(&dev_priv->hangcheck_timer,
1858                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1859                 if (was_empty)
1860                         queue_delayed_work(dev_priv->wq,
1861                                            &dev_priv->mm.retire_work, HZ);
1862         }
1863         return 0;
1864 }
1865
1866 /**
1867  * Command execution barrier
1868  *
1869  * Ensures that all commands in the ring are finished
1870  * before signalling the CPU
1871  */
1872 static void
1873 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1874 {
1875         uint32_t flush_domains = 0;
1876
1877         /* The sampler always gets flushed on i965 (sigh) */
1878         if (INTEL_INFO(dev)->gen >= 4)
1879                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1880
1881         ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
1882 }
1883
1884 static inline void
1885 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1886 {
1887         struct drm_i915_file_private *file_priv = request->file_priv;
1888
1889         if (!file_priv)
1890                 return;
1891
1892         spin_lock(&file_priv->mm.lock);
1893         list_del(&request->client_list);
1894         request->file_priv = NULL;
1895         spin_unlock(&file_priv->mm.lock);
1896 }
1897
1898 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1899                                       struct intel_ring_buffer *ring)
1900 {
1901         while (!list_empty(&ring->request_list)) {
1902                 struct drm_i915_gem_request *request;
1903
1904                 request = list_first_entry(&ring->request_list,
1905                                            struct drm_i915_gem_request,
1906                                            list);
1907
1908                 list_del(&request->list);
1909                 i915_gem_request_remove_from_client(request);
1910                 kfree(request);
1911         }
1912
1913         while (!list_empty(&ring->active_list)) {
1914                 struct drm_i915_gem_object *obj_priv;
1915
1916                 obj_priv = list_first_entry(&ring->active_list,
1917                                             struct drm_i915_gem_object,
1918                                             ring_list);
1919
1920                 obj_priv->base.write_domain = 0;
1921                 list_del_init(&obj_priv->gpu_write_list);
1922                 i915_gem_object_move_to_inactive(&obj_priv->base);
1923         }
1924 }
1925
1926 void i915_gem_reset(struct drm_device *dev)
1927 {
1928         struct drm_i915_private *dev_priv = dev->dev_private;
1929         struct drm_i915_gem_object *obj_priv;
1930         int i;
1931
1932         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1933         i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1934         i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1935
1936         /* Remove anything from the flushing lists. The GPU cache is likely
1937          * to be lost on reset along with the data, so simply move the
1938          * lost bo to the inactive list.
1939          */
1940         while (!list_empty(&dev_priv->mm.flushing_list)) {
1941                 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1942                                             struct drm_i915_gem_object,
1943                                             mm_list);
1944
1945                 obj_priv->base.write_domain = 0;
1946                 list_del_init(&obj_priv->gpu_write_list);
1947                 i915_gem_object_move_to_inactive(&obj_priv->base);
1948         }
1949
1950         /* Move everything out of the GPU domains to ensure we do any
1951          * necessary invalidation upon reuse.
1952          */
1953         list_for_each_entry(obj_priv,
1954                             &dev_priv->mm.inactive_list,
1955                             mm_list)
1956         {
1957                 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1958         }
1959
1960         /* The fence registers are invalidated so clear them out */
1961         for (i = 0; i < 16; i++) {
1962                 struct drm_i915_fence_reg *reg;
1963
1964                 reg = &dev_priv->fence_regs[i];
1965                 if (!reg->obj)
1966                         continue;
1967
1968                 i915_gem_clear_fence_reg(reg->obj);
1969         }
1970 }
1971
1972 /**
1973  * This function clears the request list as sequence numbers are passed.
1974  */
1975 static void
1976 i915_gem_retire_requests_ring(struct drm_device *dev,
1977                               struct intel_ring_buffer *ring)
1978 {
1979         drm_i915_private_t *dev_priv = dev->dev_private;
1980         uint32_t seqno;
1981
1982         if (!ring->status_page.page_addr ||
1983             list_empty(&ring->request_list))
1984                 return;
1985
1986         WARN_ON(i915_verify_lists(dev));
1987
1988         seqno = ring->get_seqno(ring);
1989         while (!list_empty(&ring->request_list)) {
1990                 struct drm_i915_gem_request *request;
1991
1992                 request = list_first_entry(&ring->request_list,
1993                                            struct drm_i915_gem_request,
1994                                            list);
1995
1996                 if (!i915_seqno_passed(seqno, request->seqno))
1997                         break;
1998
1999                 trace_i915_gem_request_retire(dev, request->seqno);
2000
2001                 list_del(&request->list);
2002                 i915_gem_request_remove_from_client(request);
2003                 kfree(request);
2004         }
2005
2006         /* Move any buffers on the active list that are no longer referenced
2007          * by the ringbuffer to the flushing/inactive lists as appropriate.
2008          */
2009         while (!list_empty(&ring->active_list)) {
2010                 struct drm_gem_object *obj;
2011                 struct drm_i915_gem_object *obj_priv;
2012
2013                 obj_priv = list_first_entry(&ring->active_list,
2014                                             struct drm_i915_gem_object,
2015                                             ring_list);
2016
2017                 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
2018                         break;
2019
2020                 obj = &obj_priv->base;
2021                 if (obj->write_domain != 0)
2022                         i915_gem_object_move_to_flushing(obj);
2023                 else
2024                         i915_gem_object_move_to_inactive(obj);
2025         }
2026
2027         if (unlikely (dev_priv->trace_irq_seqno &&
2028                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
2029                 ring->user_irq_put(ring);
2030                 dev_priv->trace_irq_seqno = 0;
2031         }
2032
2033         WARN_ON(i915_verify_lists(dev));
2034 }
2035
2036 void
2037 i915_gem_retire_requests(struct drm_device *dev)
2038 {
2039         drm_i915_private_t *dev_priv = dev->dev_private;
2040
2041         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2042             struct drm_i915_gem_object *obj_priv, *tmp;
2043
2044             /* We must be careful that during unbind() we do not
2045              * accidentally infinitely recurse into retire requests.
2046              * Currently:
2047              *   retire -> free -> unbind -> wait -> retire_ring
2048              */
2049             list_for_each_entry_safe(obj_priv, tmp,
2050                                      &dev_priv->mm.deferred_free_list,
2051                                      mm_list)
2052                     i915_gem_free_object_tail(&obj_priv->base);
2053         }
2054
2055         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
2056         i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
2057         i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
2058 }
2059
2060 static void
2061 i915_gem_retire_work_handler(struct work_struct *work)
2062 {
2063         drm_i915_private_t *dev_priv;
2064         struct drm_device *dev;
2065
2066         dev_priv = container_of(work, drm_i915_private_t,
2067                                 mm.retire_work.work);
2068         dev = dev_priv->dev;
2069
2070         /* Come back later if the device is busy... */
2071         if (!mutex_trylock(&dev->struct_mutex)) {
2072                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2073                 return;
2074         }
2075
2076         i915_gem_retire_requests(dev);
2077
2078         if (!dev_priv->mm.suspended &&
2079                 (!list_empty(&dev_priv->render_ring.request_list) ||
2080                  !list_empty(&dev_priv->bsd_ring.request_list) ||
2081                  !list_empty(&dev_priv->blt_ring.request_list)))
2082                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2083         mutex_unlock(&dev->struct_mutex);
2084 }
2085
2086 int
2087 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
2088                      bool interruptible, struct intel_ring_buffer *ring)
2089 {
2090         drm_i915_private_t *dev_priv = dev->dev_private;
2091         u32 ier;
2092         int ret = 0;
2093
2094         BUG_ON(seqno == 0);
2095
2096         if (atomic_read(&dev_priv->mm.wedged))
2097                 return -EAGAIN;
2098
2099         if (seqno == ring->outstanding_lazy_request) {
2100                 struct drm_i915_gem_request *request;
2101
2102                 request = kzalloc(sizeof(*request), GFP_KERNEL);
2103                 if (request == NULL)
2104                         return -ENOMEM;
2105
2106                 ret = i915_add_request(dev, NULL, request, ring);
2107                 if (ret) {
2108                         kfree(request);
2109                         return ret;
2110                 }
2111
2112                 seqno = request->seqno;
2113         }
2114
2115         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2116                 if (HAS_PCH_SPLIT(dev))
2117                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2118                 else
2119                         ier = I915_READ(IER);
2120                 if (!ier) {
2121                         DRM_ERROR("something (likely vbetool) disabled "
2122                                   "interrupts, re-enabling\n");
2123                         i915_driver_irq_preinstall(dev);
2124                         i915_driver_irq_postinstall(dev);
2125                 }
2126
2127                 trace_i915_gem_request_wait_begin(dev, seqno);
2128
2129                 ring->waiting_seqno = seqno;
2130                 ring->user_irq_get(ring);
2131                 if (interruptible)
2132                         ret = wait_event_interruptible(ring->irq_queue,
2133                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2134                                 || atomic_read(&dev_priv->mm.wedged));
2135                 else
2136                         wait_event(ring->irq_queue,
2137                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2138                                 || atomic_read(&dev_priv->mm.wedged));
2139
2140                 ring->user_irq_put(ring);
2141                 ring->waiting_seqno = 0;
2142
2143                 trace_i915_gem_request_wait_end(dev, seqno);
2144         }
2145         if (atomic_read(&dev_priv->mm.wedged))
2146                 ret = -EAGAIN;
2147
2148         if (ret && ret != -ERESTARTSYS)
2149                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2150                           __func__, ret, seqno, ring->get_seqno(ring),
2151                           dev_priv->next_seqno);
2152
2153         /* Directly dispatch request retiring.  While we have the work queue
2154          * to handle this, the waiter on a request often wants an associated
2155          * buffer to have made it to the inactive list, and we would need
2156          * a separate wait queue to handle that.
2157          */
2158         if (ret == 0)
2159                 i915_gem_retire_requests_ring(dev, ring);
2160
2161         return ret;
2162 }
2163
2164 /**
2165  * Waits for a sequence number to be signaled, and cleans up the
2166  * request and object lists appropriately for that event.
2167  */
2168 static int
2169 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2170                   struct intel_ring_buffer *ring)
2171 {
2172         return i915_do_wait_request(dev, seqno, 1, ring);
2173 }
2174
2175 static void
2176 i915_gem_flush_ring(struct drm_device *dev,
2177                     struct drm_file *file_priv,
2178                     struct intel_ring_buffer *ring,
2179                     uint32_t invalidate_domains,
2180                     uint32_t flush_domains)
2181 {
2182         ring->flush(ring, invalidate_domains, flush_domains);
2183         i915_gem_process_flushing_list(dev, flush_domains, ring);
2184 }
2185
2186 static void
2187 i915_gem_flush(struct drm_device *dev,
2188                struct drm_file *file_priv,
2189                uint32_t invalidate_domains,
2190                uint32_t flush_domains,
2191                uint32_t flush_rings)
2192 {
2193         drm_i915_private_t *dev_priv = dev->dev_private;
2194
2195         if (flush_domains & I915_GEM_DOMAIN_CPU)
2196                 drm_agp_chipset_flush(dev);
2197
2198         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2199                 if (flush_rings & RING_RENDER)
2200                         i915_gem_flush_ring(dev, file_priv,
2201                                             &dev_priv->render_ring,
2202                                             invalidate_domains, flush_domains);
2203                 if (flush_rings & RING_BSD)
2204                         i915_gem_flush_ring(dev, file_priv,
2205                                             &dev_priv->bsd_ring,
2206                                             invalidate_domains, flush_domains);
2207                 if (flush_rings & RING_BLT)
2208                         i915_gem_flush_ring(dev, file_priv,
2209                                             &dev_priv->blt_ring,
2210                                             invalidate_domains, flush_domains);
2211         }
2212 }
2213
2214 /**
2215  * Ensures that all rendering to the object has completed and the object is
2216  * safe to unbind from the GTT or access from the CPU.
2217  */
2218 static int
2219 i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2220                                bool interruptible)
2221 {
2222         struct drm_device *dev = obj->dev;
2223         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2224         int ret;
2225
2226         /* This function only exists to support waiting for existing rendering,
2227          * not for emitting required flushes.
2228          */
2229         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2230
2231         /* If there is rendering queued on the buffer being evicted, wait for
2232          * it.
2233          */
2234         if (obj_priv->active) {
2235                 ret = i915_do_wait_request(dev,
2236                                            obj_priv->last_rendering_seqno,
2237                                            interruptible,
2238                                            obj_priv->ring);
2239                 if (ret)
2240                         return ret;
2241         }
2242
2243         return 0;
2244 }
2245
2246 /**
2247  * Unbinds an object from the GTT aperture.
2248  */
2249 int
2250 i915_gem_object_unbind(struct drm_gem_object *obj)
2251 {
2252         struct drm_device *dev = obj->dev;
2253         struct drm_i915_private *dev_priv = dev->dev_private;
2254         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2255         int ret = 0;
2256
2257         if (obj_priv->gtt_space == NULL)
2258                 return 0;
2259
2260         if (obj_priv->pin_count != 0) {
2261                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2262                 return -EINVAL;
2263         }
2264
2265         /* blow away mappings if mapped through GTT */
2266         i915_gem_release_mmap(obj);
2267
2268         /* Move the object to the CPU domain to ensure that
2269          * any possible CPU writes while it's not in the GTT
2270          * are flushed when we go to remap it. This will
2271          * also ensure that all pending GPU writes are finished
2272          * before we unbind.
2273          */
2274         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2275         if (ret == -ERESTARTSYS)
2276                 return ret;
2277         /* Continue on if we fail due to EIO, the GPU is hung so we
2278          * should be safe and we need to cleanup or else we might
2279          * cause memory corruption through use-after-free.
2280          */
2281         if (ret) {
2282                 i915_gem_clflush_object(obj);
2283                 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2284         }
2285
2286         /* release the fence reg _after_ flushing */
2287         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2288                 i915_gem_clear_fence_reg(obj);
2289
2290         drm_unbind_agp(obj_priv->agp_mem);
2291         drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2292
2293         i915_gem_object_put_pages_gtt(obj);
2294
2295         i915_gem_info_remove_gtt(dev_priv, obj_priv);
2296         list_del_init(&obj_priv->mm_list);
2297         /* Avoid an unnecessary call to unbind on rebind. */
2298         obj_priv->map_and_fenceable = true;
2299
2300         drm_mm_put_block(obj_priv->gtt_space);
2301         obj_priv->gtt_space = NULL;
2302         obj_priv->gtt_offset = 0;
2303
2304         if (i915_gem_object_is_purgeable(obj_priv))
2305                 i915_gem_object_truncate(obj);
2306
2307         trace_i915_gem_object_unbind(obj);
2308
2309         return ret;
2310 }
2311
2312 static int i915_ring_idle(struct drm_device *dev,
2313                           struct intel_ring_buffer *ring)
2314 {
2315         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2316                 return 0;
2317
2318         i915_gem_flush_ring(dev, NULL, ring,
2319                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2320         return i915_wait_request(dev,
2321                                  i915_gem_next_request_seqno(dev, ring),
2322                                  ring);
2323 }
2324
2325 int
2326 i915_gpu_idle(struct drm_device *dev)
2327 {
2328         drm_i915_private_t *dev_priv = dev->dev_private;
2329         bool lists_empty;
2330         int ret;
2331
2332         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2333                        list_empty(&dev_priv->mm.active_list));
2334         if (lists_empty)
2335                 return 0;
2336
2337         /* Flush everything onto the inactive list. */
2338         ret = i915_ring_idle(dev, &dev_priv->render_ring);
2339         if (ret)
2340                 return ret;
2341
2342         ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2343         if (ret)
2344                 return ret;
2345
2346         ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2347         if (ret)
2348                 return ret;
2349
2350         return 0;
2351 }
2352
2353 static void sandybridge_write_fence_reg(struct drm_gem_object *obj)
2354 {
2355         struct drm_device *dev = obj->dev;
2356         drm_i915_private_t *dev_priv = dev->dev_private;
2357         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2358         u32 size = i915_gem_get_gtt_size(obj_priv);
2359         int regnum = obj_priv->fence_reg;
2360         uint64_t val;
2361
2362         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2363                     0xfffff000) << 32;
2364         val |= obj_priv->gtt_offset & 0xfffff000;
2365         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2366                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2367
2368         if (obj_priv->tiling_mode == I915_TILING_Y)
2369                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2370         val |= I965_FENCE_REG_VALID;
2371
2372         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2373 }
2374
2375 static void i965_write_fence_reg(struct drm_gem_object *obj)
2376 {
2377         struct drm_device *dev = obj->dev;
2378         drm_i915_private_t *dev_priv = dev->dev_private;
2379         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2380         u32 size = i915_gem_get_gtt_size(obj_priv);
2381         int regnum = obj_priv->fence_reg;
2382         uint64_t val;
2383
2384         val = (uint64_t)((obj_priv->gtt_offset + size - 4096) &
2385                     0xfffff000) << 32;
2386         val |= obj_priv->gtt_offset & 0xfffff000;
2387         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2388         if (obj_priv->tiling_mode == I915_TILING_Y)
2389                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2390         val |= I965_FENCE_REG_VALID;
2391
2392         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2393 }
2394
2395 static void i915_write_fence_reg(struct drm_gem_object *obj)
2396 {
2397         struct drm_device *dev = obj->dev;
2398         drm_i915_private_t *dev_priv = dev->dev_private;
2399         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2400         u32 size = i915_gem_get_gtt_size(obj_priv);
2401         uint32_t fence_reg, val, pitch_val;
2402         int tile_width;
2403
2404         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2405             (obj_priv->gtt_offset & (size - 1))) {
2406                 WARN(1, "%s: object 0x%08x [fenceable? %d] not 1M or size (0x%08x) aligned [gtt_space offset=%lx, size=%lx]\n",
2407                      __func__, obj_priv->gtt_offset, obj_priv->map_and_fenceable, size,
2408                      obj_priv->gtt_space->start, obj_priv->gtt_space->size);
2409                 return;
2410         }
2411
2412         if (obj_priv->tiling_mode == I915_TILING_Y &&
2413             HAS_128_BYTE_Y_TILING(dev))
2414                 tile_width = 128;
2415         else
2416                 tile_width = 512;
2417
2418         /* Note: pitch better be a power of two tile widths */
2419         pitch_val = obj_priv->stride / tile_width;
2420         pitch_val = ffs(pitch_val) - 1;
2421
2422         if (obj_priv->tiling_mode == I915_TILING_Y &&
2423             HAS_128_BYTE_Y_TILING(dev))
2424                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2425         else
2426                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2427
2428         val = obj_priv->gtt_offset;
2429         if (obj_priv->tiling_mode == I915_TILING_Y)
2430                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2431         val |= I915_FENCE_SIZE_BITS(size);
2432         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2433         val |= I830_FENCE_REG_VALID;
2434
2435         fence_reg = obj_priv->fence_reg;
2436         if (fence_reg < 8)
2437                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2438         else
2439                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2440         I915_WRITE(fence_reg, val);
2441 }
2442
2443 static void i830_write_fence_reg(struct drm_gem_object *obj)
2444 {
2445         struct drm_device *dev = obj->dev;
2446         drm_i915_private_t *dev_priv = dev->dev_private;
2447         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2448         u32 size = i915_gem_get_gtt_size(obj_priv);
2449         int regnum = obj_priv->fence_reg;
2450         uint32_t val;
2451         uint32_t pitch_val;
2452         uint32_t fence_size_bits;
2453
2454         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2455             (obj_priv->gtt_offset & (obj->size - 1))) {
2456                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2457                      __func__, obj_priv->gtt_offset);
2458                 return;
2459         }
2460
2461         pitch_val = obj_priv->stride / 128;
2462         pitch_val = ffs(pitch_val) - 1;
2463         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2464
2465         val = obj_priv->gtt_offset;
2466         if (obj_priv->tiling_mode == I915_TILING_Y)
2467                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2468         fence_size_bits = I830_FENCE_SIZE_BITS(size);
2469         WARN_ON(fence_size_bits & ~0x00000f00);
2470         val |= fence_size_bits;
2471         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2472         val |= I830_FENCE_REG_VALID;
2473
2474         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2475 }
2476
2477 static int i915_find_fence_reg(struct drm_device *dev,
2478                                bool interruptible)
2479 {
2480         struct drm_i915_private *dev_priv = dev->dev_private;
2481         struct drm_i915_fence_reg *reg;
2482         struct drm_i915_gem_object *obj_priv = NULL;
2483         int i, avail, ret;
2484
2485         /* First try to find a free reg */
2486         avail = 0;
2487         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2488                 reg = &dev_priv->fence_regs[i];
2489                 if (!reg->obj)
2490                         return i;
2491
2492                 obj_priv = to_intel_bo(reg->obj);
2493                 if (!obj_priv->pin_count)
2494                     avail++;
2495         }
2496
2497         if (avail == 0)
2498                 return -ENOSPC;
2499
2500         /* None available, try to steal one or wait for a user to finish */
2501         avail = I915_FENCE_REG_NONE;
2502         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2503                             lru_list) {
2504                 obj_priv = to_intel_bo(reg->obj);
2505                 if (obj_priv->pin_count)
2506                         continue;
2507
2508                 /* found one! */
2509                 avail = obj_priv->fence_reg;
2510                 break;
2511         }
2512
2513         BUG_ON(avail == I915_FENCE_REG_NONE);
2514
2515         /* We only have a reference on obj from the active list. put_fence_reg
2516          * might drop that one, causing a use-after-free in it. So hold a
2517          * private reference to obj like the other callers of put_fence_reg
2518          * (set_tiling ioctl) do. */
2519         drm_gem_object_reference(&obj_priv->base);
2520         ret = i915_gem_object_put_fence_reg(&obj_priv->base, interruptible);
2521         drm_gem_object_unreference(&obj_priv->base);
2522         if (ret != 0)
2523                 return ret;
2524
2525         return avail;
2526 }
2527
2528 /**
2529  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2530  * @obj: object to map through a fence reg
2531  *
2532  * When mapping objects through the GTT, userspace wants to be able to write
2533  * to them without having to worry about swizzling if the object is tiled.
2534  *
2535  * This function walks the fence regs looking for a free one for @obj,
2536  * stealing one if it can't find any.
2537  *
2538  * It then sets up the reg based on the object's properties: address, pitch
2539  * and tiling format.
2540  */
2541 int
2542 i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2543                               bool interruptible)
2544 {
2545         struct drm_device *dev = obj->dev;
2546         struct drm_i915_private *dev_priv = dev->dev_private;
2547         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2548         struct drm_i915_fence_reg *reg = NULL;
2549         int ret;
2550
2551         /* Just update our place in the LRU if our fence is getting used. */
2552         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2553                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2554                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2555                 return 0;
2556         }
2557
2558         switch (obj_priv->tiling_mode) {
2559         case I915_TILING_NONE:
2560                 WARN(1, "allocating a fence for non-tiled object?\n");
2561                 break;
2562         case I915_TILING_X:
2563                 if (!obj_priv->stride)
2564                         return -EINVAL;
2565                 WARN((obj_priv->stride & (512 - 1)),
2566                      "object 0x%08x is X tiled but has non-512B pitch\n",
2567                      obj_priv->gtt_offset);
2568                 break;
2569         case I915_TILING_Y:
2570                 if (!obj_priv->stride)
2571                         return -EINVAL;
2572                 WARN((obj_priv->stride & (128 - 1)),
2573                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2574                      obj_priv->gtt_offset);
2575                 break;
2576         }
2577
2578         ret = i915_find_fence_reg(dev, interruptible);
2579         if (ret < 0)
2580                 return ret;
2581
2582         obj_priv->fence_reg = ret;
2583         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2584         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2585
2586         reg->obj = obj;
2587
2588         switch (INTEL_INFO(dev)->gen) {
2589         case 6:
2590                 sandybridge_write_fence_reg(obj);
2591                 break;
2592         case 5:
2593         case 4:
2594                 i965_write_fence_reg(obj);
2595                 break;
2596         case 3:
2597                 i915_write_fence_reg(obj);
2598                 break;
2599         case 2:
2600                 i830_write_fence_reg(obj);
2601                 break;
2602         }
2603
2604         trace_i915_gem_object_get_fence(obj,
2605                                         obj_priv->fence_reg,
2606                                         obj_priv->tiling_mode);
2607
2608         return 0;
2609 }
2610
2611 /**
2612  * i915_gem_clear_fence_reg - clear out fence register info
2613  * @obj: object to clear
2614  *
2615  * Zeroes out the fence register itself and clears out the associated
2616  * data structures in dev_priv and obj_priv.
2617  */
2618 static void
2619 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2620 {
2621         struct drm_device *dev = obj->dev;
2622         drm_i915_private_t *dev_priv = dev->dev_private;
2623         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2624         struct drm_i915_fence_reg *reg =
2625                 &dev_priv->fence_regs[obj_priv->fence_reg];
2626         uint32_t fence_reg;
2627
2628         switch (INTEL_INFO(dev)->gen) {
2629         case 6:
2630                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2631                              (obj_priv->fence_reg * 8), 0);
2632                 break;
2633         case 5:
2634         case 4:
2635                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2636                 break;
2637         case 3:
2638                 if (obj_priv->fence_reg >= 8)
2639                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2640                 else
2641         case 2:
2642                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2643
2644                 I915_WRITE(fence_reg, 0);
2645                 break;
2646         }
2647
2648         reg->obj = NULL;
2649         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2650         list_del_init(&reg->lru_list);
2651 }
2652
2653 /**
2654  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2655  * to the buffer to finish, and then resets the fence register.
2656  * @obj: tiled object holding a fence register.
2657  * @bool: whether the wait upon the fence is interruptible
2658  *
2659  * Zeroes out the fence register itself and clears out the associated
2660  * data structures in dev_priv and obj_priv.
2661  */
2662 int
2663 i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2664                               bool interruptible)
2665 {
2666         struct drm_device *dev = obj->dev;
2667         struct drm_i915_private *dev_priv = dev->dev_private;
2668         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2669         struct drm_i915_fence_reg *reg;
2670
2671         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2672                 return 0;
2673
2674         /* If we've changed tiling, GTT-mappings of the object
2675          * need to re-fault to ensure that the correct fence register
2676          * setup is in place.
2677          */
2678         i915_gem_release_mmap(obj);
2679
2680         /* On the i915, GPU access to tiled buffers is via a fence,
2681          * therefore we must wait for any outstanding access to complete
2682          * before clearing the fence.
2683          */
2684         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2685         if (reg->gpu) {
2686                 int ret;
2687
2688                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2689                 if (ret)
2690                         return ret;
2691
2692                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2693                 if (ret)
2694                         return ret;
2695
2696                 reg->gpu = false;
2697         }
2698
2699         i915_gem_object_flush_gtt_write_domain(obj);
2700         i915_gem_clear_fence_reg(obj);
2701
2702         return 0;
2703 }
2704
2705 /**
2706  * Finds free space in the GTT aperture and binds the object there.
2707  */
2708 static int
2709 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
2710                             unsigned alignment,
2711                             bool map_and_fenceable)
2712 {
2713         struct drm_device *dev = obj->dev;
2714         drm_i915_private_t *dev_priv = dev->dev_private;
2715         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2716         struct drm_mm_node *free_space;
2717         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2718         u32 size, fence_size, fence_alignment, unfenced_alignment;
2719         bool mappable, fenceable;
2720         int ret;
2721
2722         if (obj_priv->madv != I915_MADV_WILLNEED) {
2723                 DRM_ERROR("Attempting to bind a purgeable object\n");
2724                 return -EINVAL;
2725         }
2726
2727         fence_size = i915_gem_get_gtt_size(obj_priv);
2728         fence_alignment = i915_gem_get_gtt_alignment(obj_priv);
2729         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj_priv);
2730
2731         if (alignment == 0)
2732                 alignment = map_and_fenceable ? fence_alignment :
2733                                                 unfenced_alignment;
2734         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2735                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2736                 return -EINVAL;
2737         }
2738
2739         size = map_and_fenceable ? fence_size : obj->size;
2740
2741         /* If the object is bigger than the entire aperture, reject it early
2742          * before evicting everything in a vain attempt to find space.
2743          */
2744         if (obj->size >
2745             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2746                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2747                 return -E2BIG;
2748         }
2749
2750  search_free:
2751         if (map_and_fenceable)
2752                 free_space =
2753                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2754                                                     size, alignment, 0,
2755                                                     dev_priv->mm.gtt_mappable_end,
2756                                                     0);
2757         else
2758                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2759                                                 size, alignment, 0);
2760
2761         if (free_space != NULL) {
2762                 if (map_and_fenceable)
2763                         obj_priv->gtt_space =
2764                                 drm_mm_get_block_range_generic(free_space,
2765                                                                size, alignment, 0,
2766                                                                dev_priv->mm.gtt_mappable_end,
2767                                                                0);
2768                 else
2769                         obj_priv->gtt_space =
2770                                 drm_mm_get_block(free_space, size, alignment);
2771         }
2772         if (obj_priv->gtt_space == NULL) {
2773                 /* If the gtt is empty and we're still having trouble
2774                  * fitting our object in, we're out of memory.
2775                  */
2776                 ret = i915_gem_evict_something(dev, size, alignment,
2777                                                map_and_fenceable);
2778                 if (ret)
2779                         return ret;
2780
2781                 goto search_free;
2782         }
2783
2784         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2785         if (ret) {
2786                 drm_mm_put_block(obj_priv->gtt_space);
2787                 obj_priv->gtt_space = NULL;
2788
2789                 if (ret == -ENOMEM) {
2790                         /* first try to clear up some space from the GTT */
2791                         ret = i915_gem_evict_something(dev, size,
2792                                                        alignment,
2793                                                        map_and_fenceable);
2794                         if (ret) {
2795                                 /* now try to shrink everyone else */
2796                                 if (gfpmask) {
2797                                         gfpmask = 0;
2798                                         goto search_free;
2799                                 }
2800
2801                                 return ret;
2802                         }
2803
2804                         goto search_free;
2805                 }
2806
2807                 return ret;
2808         }
2809
2810         /* Create an AGP memory structure pointing at our pages, and bind it
2811          * into the GTT.
2812          */
2813         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2814                                                obj_priv->pages,
2815                                                obj->size >> PAGE_SHIFT,
2816                                                obj_priv->gtt_space->start,
2817                                                obj_priv->agp_type);
2818         if (obj_priv->agp_mem == NULL) {
2819                 i915_gem_object_put_pages_gtt(obj);
2820                 drm_mm_put_block(obj_priv->gtt_space);
2821                 obj_priv->gtt_space = NULL;
2822
2823                 ret = i915_gem_evict_something(dev, size,
2824                                                alignment, map_and_fenceable);
2825                 if (ret)
2826                         return ret;
2827
2828                 goto search_free;
2829         }
2830
2831         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2832
2833         /* keep track of bounds object by adding it to the inactive list */
2834         list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list);
2835         i915_gem_info_add_gtt(dev_priv, obj_priv);
2836
2837         /* Assert that the object is not currently in any GPU domain. As it
2838          * wasn't in the GTT, there shouldn't be any way it could have been in
2839          * a GPU cache
2840          */
2841         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2842         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2843
2844         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset, map_and_fenceable);
2845
2846         fenceable =
2847                 obj_priv->gtt_space->size == fence_size &&
2848                 (obj_priv->gtt_space->start & (fence_alignment -1)) == 0;
2849
2850         mappable =
2851                 obj_priv->gtt_offset + obj->size <= dev_priv->mm.gtt_mappable_end;
2852
2853         obj_priv->map_and_fenceable = mappable && fenceable;
2854
2855         return 0;
2856 }
2857
2858 void
2859 i915_gem_clflush_object(struct drm_gem_object *obj)
2860 {
2861         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2862
2863         /* If we don't have a page list set up, then we're not pinned
2864          * to GPU, and we can ignore the cache flush because it'll happen
2865          * again at bind time.
2866          */
2867         if (obj_priv->pages == NULL)
2868                 return;
2869
2870         trace_i915_gem_object_clflush(obj);
2871
2872         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2873 }
2874
2875 /** Flushes any GPU write domain for the object if it's dirty. */
2876 static int
2877 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2878                                        bool pipelined)
2879 {
2880         struct drm_device *dev = obj->dev;
2881
2882         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2883                 return 0;
2884
2885         /* Queue the GPU write cache flushing we need. */
2886         i915_gem_flush_ring(dev, NULL,
2887                             to_intel_bo(obj)->ring,
2888                             0, obj->write_domain);
2889         BUG_ON(obj->write_domain);
2890
2891         if (pipelined)
2892                 return 0;
2893
2894         return i915_gem_object_wait_rendering(obj, true);
2895 }
2896
2897 /** Flushes the GTT write domain for the object if it's dirty. */
2898 static void
2899 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2900 {
2901         uint32_t old_write_domain;
2902
2903         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2904                 return;
2905
2906         /* No actual flushing is required for the GTT write domain.   Writes
2907          * to it immediately go to main memory as far as we know, so there's
2908          * no chipset flush.  It also doesn't land in render cache.
2909          */
2910         i915_gem_release_mmap(obj);
2911
2912         old_write_domain = obj->write_domain;
2913         obj->write_domain = 0;
2914
2915         trace_i915_gem_object_change_domain(obj,
2916                                             obj->read_domains,
2917                                             old_write_domain);
2918 }
2919
2920 /** Flushes the CPU write domain for the object if it's dirty. */
2921 static void
2922 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2923 {
2924         struct drm_device *dev = obj->dev;
2925         uint32_t old_write_domain;
2926
2927         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2928                 return;
2929
2930         i915_gem_clflush_object(obj);
2931         drm_agp_chipset_flush(dev);
2932         old_write_domain = obj->write_domain;
2933         obj->write_domain = 0;
2934
2935         trace_i915_gem_object_change_domain(obj,
2936                                             obj->read_domains,
2937                                             old_write_domain);
2938 }
2939
2940 /**
2941  * Moves a single object to the GTT read, and possibly write domain.
2942  *
2943  * This function returns when the move is complete, including waiting on
2944  * flushes to occur.
2945  */
2946 int
2947 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2948 {
2949         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2950         uint32_t old_write_domain, old_read_domains;
2951         int ret;
2952
2953         /* Not valid to be called on unbound objects. */
2954         if (obj_priv->gtt_space == NULL)
2955                 return -EINVAL;
2956
2957         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2958         if (ret != 0)
2959                 return ret;
2960
2961         i915_gem_object_flush_cpu_write_domain(obj);
2962
2963         if (write) {
2964                 ret = i915_gem_object_wait_rendering(obj, true);
2965                 if (ret)
2966                         return ret;
2967         }
2968
2969         old_write_domain = obj->write_domain;
2970         old_read_domains = obj->read_domains;
2971
2972         /* It should now be out of any other write domains, and we can update
2973          * the domain values for our changes.
2974          */
2975         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2976         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2977         if (write) {
2978                 obj->read_domains = I915_GEM_DOMAIN_GTT;
2979                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2980                 obj_priv->dirty = 1;
2981         }
2982
2983         trace_i915_gem_object_change_domain(obj,
2984                                             old_read_domains,
2985                                             old_write_domain);
2986
2987         return 0;
2988 }
2989
2990 /*
2991  * Prepare buffer for display plane. Use uninterruptible for possible flush
2992  * wait, as in modesetting process we're not supposed to be interrupted.
2993  */
2994 int
2995 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2996                                      bool pipelined)
2997 {
2998         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2999         uint32_t old_read_domains;
3000         int ret;
3001
3002         /* Not valid to be called on unbound objects. */
3003         if (obj_priv->gtt_space == NULL)
3004                 return -EINVAL;
3005
3006         ret = i915_gem_object_flush_gpu_write_domain(obj, true);
3007         if (ret)
3008                 return ret;
3009
3010         /* Currently, we are always called from an non-interruptible context. */
3011         if (!pipelined) {
3012                 ret = i915_gem_object_wait_rendering(obj, false);
3013                 if (ret)
3014                         return ret;
3015         }
3016
3017         i915_gem_object_flush_cpu_write_domain(obj);
3018
3019         old_read_domains = obj->read_domains;
3020         obj->read_domains |= I915_GEM_DOMAIN_GTT;
3021
3022         trace_i915_gem_object_change_domain(obj,
3023                                             old_read_domains,
3024                                             obj->write_domain);
3025
3026         return 0;
3027 }
3028
3029 int
3030 i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
3031                           bool interruptible)
3032 {
3033         if (!obj->active)
3034                 return 0;
3035
3036         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
3037                 i915_gem_flush_ring(obj->base.dev, NULL, obj->ring,
3038                                     0, obj->base.write_domain);
3039
3040         return i915_gem_object_wait_rendering(&obj->base, interruptible);
3041 }
3042
3043 /**
3044  * Moves a single object to the CPU read, and possibly write domain.
3045  *
3046  * This function returns when the move is complete, including waiting on
3047  * flushes to occur.
3048  */
3049 static int
3050 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
3051 {
3052         uint32_t old_write_domain, old_read_domains;
3053         int ret;
3054
3055         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3056         if (ret != 0)
3057                 return ret;
3058
3059         i915_gem_object_flush_gtt_write_domain(obj);
3060
3061         /* If we have a partially-valid cache of the object in the CPU,
3062          * finish invalidating it and free the per-page flags.
3063          */
3064         i915_gem_object_set_to_full_cpu_read_domain(obj);
3065
3066         if (write) {
3067                 ret = i915_gem_object_wait_rendering(obj, true);
3068                 if (ret)
3069                         return ret;
3070         }
3071
3072         old_write_domain = obj->write_domain;
3073         old_read_domains = obj->read_domains;
3074
3075         /* Flush the CPU cache if it's still invalid. */
3076         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3077                 i915_gem_clflush_object(obj);
3078
3079                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3080         }
3081
3082         /* It should now be out of any other write domains, and we can update
3083          * the domain values for our changes.
3084          */
3085         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3086
3087         /* If we're writing through the CPU, then the GPU read domains will
3088          * need to be invalidated at next use.
3089          */
3090         if (write) {
3091                 obj->read_domains = I915_GEM_DOMAIN_CPU;
3092                 obj->write_domain = I915_GEM_DOMAIN_CPU;
3093         }
3094
3095         trace_i915_gem_object_change_domain(obj,
3096                                             old_read_domains,
3097                                             old_write_domain);
3098
3099         return 0;
3100 }
3101
3102 /*
3103  * Set the next domain for the specified object. This
3104  * may not actually perform the necessary flushing/invaliding though,
3105  * as that may want to be batched with other set_domain operations
3106  *
3107  * This is (we hope) the only really tricky part of gem. The goal
3108  * is fairly simple -- track which caches hold bits of the object
3109  * and make sure they remain coherent. A few concrete examples may
3110  * help to explain how it works. For shorthand, we use the notation
3111  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3112  * a pair of read and write domain masks.
3113  *
3114  * Case 1: the batch buffer
3115  *
3116  *      1. Allocated
3117  *      2. Written by CPU
3118  *      3. Mapped to GTT
3119  *      4. Read by GPU
3120  *      5. Unmapped from GTT
3121  *      6. Freed
3122  *
3123  *      Let's take these a step at a time
3124  *
3125  *      1. Allocated
3126  *              Pages allocated from the kernel may still have
3127  *              cache contents, so we set them to (CPU, CPU) always.
3128  *      2. Written by CPU (using pwrite)
3129  *              The pwrite function calls set_domain (CPU, CPU) and
3130  *              this function does nothing (as nothing changes)
3131  *      3. Mapped by GTT
3132  *              This function asserts that the object is not
3133  *              currently in any GPU-based read or write domains
3134  *      4. Read by GPU
3135  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3136  *              As write_domain is zero, this function adds in the
3137  *              current read domains (CPU+COMMAND, 0).
3138  *              flush_domains is set to CPU.
3139  *              invalidate_domains is set to COMMAND
3140  *              clflush is run to get data out of the CPU caches
3141  *              then i915_dev_set_domain calls i915_gem_flush to
3142  *              emit an MI_FLUSH and drm_agp_chipset_flush
3143  *      5. Unmapped from GTT
3144  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3145  *              flush_domains and invalidate_domains end up both zero
3146  *              so no flushing/invalidating happens
3147  *      6. Freed
3148  *              yay, done
3149  *
3150  * Case 2: The shared render buffer
3151  *
3152  *      1. Allocated
3153  *      2. Mapped to GTT
3154  *      3. Read/written by GPU
3155  *      4. set_domain to (CPU,CPU)
3156  *      5. Read/written by CPU
3157  *      6. Read/written by GPU
3158  *
3159  *      1. Allocated
3160  *              Same as last example, (CPU, CPU)
3161  *      2. Mapped to GTT
3162  *              Nothing changes (assertions find that it is not in the GPU)
3163  *      3. Read/written by GPU
3164  *              execbuffer calls set_domain (RENDER, RENDER)
3165  *              flush_domains gets CPU
3166  *              invalidate_domains gets GPU
3167  *              clflush (obj)
3168  *              MI_FLUSH and drm_agp_chipset_flush
3169  *      4. set_domain (CPU, CPU)
3170  *              flush_domains gets GPU
3171  *              invalidate_domains gets CPU
3172  *              wait_rendering (obj) to make sure all drawing is complete.
3173  *              This will include an MI_FLUSH to get the data from GPU
3174  *              to memory
3175  *              clflush (obj) to invalidate the CPU cache
3176  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3177  *      5. Read/written by CPU
3178  *              cache lines are loaded and dirtied
3179  *      6. Read written by GPU
3180  *              Same as last GPU access
3181  *
3182  * Case 3: The constant buffer
3183  *
3184  *      1. Allocated
3185  *      2. Written by CPU
3186  *      3. Read by GPU
3187  *      4. Updated (written) by CPU again
3188  *      5. Read by GPU
3189  *
3190  *      1. Allocated
3191  *              (CPU, CPU)
3192  *      2. Written by CPU
3193  *              (CPU, CPU)
3194  *      3. Read by GPU
3195  *              (CPU+RENDER, 0)
3196  *              flush_domains = CPU
3197  *              invalidate_domains = RENDER
3198  *              clflush (obj)
3199  *              MI_FLUSH
3200  *              drm_agp_chipset_flush
3201  *      4. Updated (written) by CPU again
3202  *              (CPU, CPU)
3203  *              flush_domains = 0 (no previous write domain)
3204  *              invalidate_domains = 0 (no new read domains)
3205  *      5. Read by GPU
3206  *              (CPU+RENDER, 0)
3207  *              flush_domains = CPU
3208  *              invalidate_domains = RENDER
3209  *              clflush (obj)
3210  *              MI_FLUSH
3211  *              drm_agp_chipset_flush
3212  */
3213 static void
3214 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
3215                                   struct intel_ring_buffer *ring,
3216                                   struct change_domains *cd)
3217 {
3218         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3219         uint32_t                        invalidate_domains = 0;
3220         uint32_t                        flush_domains = 0;
3221
3222         /*
3223          * If the object isn't moving to a new write domain,
3224          * let the object stay in multiple read domains
3225          */
3226         if (obj->pending_write_domain == 0)
3227                 obj->pending_read_domains |= obj->read_domains;
3228
3229         /*
3230          * Flush the current write domain if
3231          * the new read domains don't match. Invalidate
3232          * any read domains which differ from the old
3233          * write domain
3234          */
3235         if (obj->write_domain &&
3236             (obj->write_domain != obj->pending_read_domains ||
3237              obj_priv->ring != ring)) {
3238                 flush_domains |= obj->write_domain;
3239                 invalidate_domains |=
3240                         obj->pending_read_domains & ~obj->write_domain;
3241         }
3242         /*
3243          * Invalidate any read caches which may have
3244          * stale data. That is, any new read domains.
3245          */
3246         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3247         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3248                 i915_gem_clflush_object(obj);
3249
3250         /* blow away mappings if mapped through GTT */
3251         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
3252                 i915_gem_release_mmap(obj);
3253
3254         /* The actual obj->write_domain will be updated with
3255          * pending_write_domain after we emit the accumulated flush for all
3256          * of our domain changes in execbuffers (which clears objects'
3257          * write_domains).  So if we have a current write domain that we
3258          * aren't changing, set pending_write_domain to that.
3259          */
3260         if (flush_domains == 0 && obj->pending_write_domain == 0)
3261                 obj->pending_write_domain = obj->write_domain;
3262
3263         cd->invalidate_domains |= invalidate_domains;
3264         cd->flush_domains |= flush_domains;
3265         if (flush_domains & I915_GEM_GPU_DOMAINS)
3266                 cd->flush_rings |= obj_priv->ring->id;
3267         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3268                 cd->flush_rings |= ring->id;
3269 }
3270
3271 /**
3272  * Moves the object from a partially CPU read to a full one.
3273  *
3274  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3275  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3276  */
3277 static void
3278 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3279 {
3280         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3281
3282         if (!obj_priv->page_cpu_valid)
3283                 return;
3284
3285         /* If we're partially in the CPU read domain, finish moving it in.
3286          */
3287         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3288                 int i;
3289
3290                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3291                         if (obj_priv->page_cpu_valid[i])
3292                                 continue;
3293                         drm_clflush_pages(obj_priv->pages + i, 1);
3294                 }
3295         }
3296
3297         /* Free the page_cpu_valid mappings which are now stale, whether
3298          * or not we've got I915_GEM_DOMAIN_CPU.
3299          */
3300         kfree(obj_priv->page_cpu_valid);
3301         obj_priv->page_cpu_valid = NULL;
3302 }
3303
3304 /**
3305  * Set the CPU read domain on a range of the object.
3306  *
3307  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3308  * not entirely valid.  The page_cpu_valid member of the object flags which
3309  * pages have been flushed, and will be respected by
3310  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3311  * of the whole object.
3312  *
3313  * This function returns when the move is complete, including waiting on
3314  * flushes to occur.
3315  */
3316 static int
3317 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3318                                           uint64_t offset, uint64_t size)
3319 {
3320         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3321         uint32_t old_read_domains;
3322         int i, ret;
3323
3324         if (offset == 0 && size == obj->size)
3325                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3326
3327         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3328         if (ret != 0)
3329                 return ret;
3330         i915_gem_object_flush_gtt_write_domain(obj);
3331
3332         /* If we're already fully in the CPU read domain, we're done. */
3333         if (obj_priv->page_cpu_valid == NULL &&
3334             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3335                 return 0;
3336
3337         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3338          * newly adding I915_GEM_DOMAIN_CPU
3339          */
3340         if (obj_priv->page_cpu_valid == NULL) {
3341                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3342                                                    GFP_KERNEL);
3343                 if (obj_priv->page_cpu_valid == NULL)
3344                         return -ENOMEM;
3345         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3346                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3347
3348         /* Flush the cache on any pages that are still invalid from the CPU's
3349          * perspective.
3350          */
3351         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3352              i++) {
3353                 if (obj_priv->page_cpu_valid[i])
3354                         continue;
3355
3356                 drm_clflush_pages(obj_priv->pages + i, 1);
3357
3358                 obj_priv->page_cpu_valid[i] = 1;
3359         }
3360
3361         /* It should now be out of any other write domains, and we can update
3362          * the domain values for our changes.
3363          */
3364         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3365
3366         old_read_domains = obj->read_domains;
3367         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3368
3369         trace_i915_gem_object_change_domain(obj,
3370                                             old_read_domains,
3371                                             obj->write_domain);
3372
3373         return 0;
3374 }
3375
3376 /**
3377  * Pin an object to the GTT and evaluate the relocations landing in it.
3378  */
3379 static int
3380 i915_gem_execbuffer_relocate(struct drm_i915_gem_object *obj,
3381                              struct drm_file *file_priv,
3382                              struct drm_i915_gem_exec_object2 *entry)
3383 {
3384         struct drm_device *dev = obj->base.dev;
3385         drm_i915_private_t *dev_priv = dev->dev_private;
3386         struct drm_i915_gem_relocation_entry __user *user_relocs;
3387         struct drm_gem_object *target_obj = NULL;
3388         uint32_t target_handle = 0;
3389         int i, ret = 0;
3390
3391         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3392         for (i = 0; i < entry->relocation_count; i++) {
3393                 struct drm_i915_gem_relocation_entry reloc;
3394                 uint32_t target_offset;
3395
3396                 if (__copy_from_user_inatomic(&reloc,
3397                                               user_relocs+i,
3398                                               sizeof(reloc))) {
3399                         ret = -EFAULT;
3400                         break;
3401                 }
3402
3403                 if (reloc.target_handle != target_handle) {
3404                         drm_gem_object_unreference(target_obj);
3405
3406                         target_obj = drm_gem_object_lookup(dev, file_priv,
3407                                                            reloc.target_handle);
3408                         if (target_obj == NULL) {
3409                                 ret = -ENOENT;
3410                                 break;
3411                         }
3412
3413                         target_handle = reloc.target_handle;
3414                 }
3415                 target_offset = to_intel_bo(target_obj)->gtt_offset;
3416
3417 #if WATCH_RELOC
3418                 DRM_INFO("%s: obj %p offset %08x target %d "
3419                          "read %08x write %08x gtt %08x "
3420                          "presumed %08x delta %08x\n",
3421                          __func__,
3422                          obj,
3423                          (int) reloc.offset,
3424                          (int) reloc.target_handle,
3425                          (int) reloc.read_domains,
3426                          (int) reloc.write_domain,
3427                          (int) target_offset,
3428                          (int) reloc.presumed_offset,
3429                          reloc.delta);
3430 #endif
3431
3432                 /* The target buffer should have appeared before us in the
3433                  * exec_object list, so it should have a GTT space bound by now.
3434                  */
3435                 if (target_offset == 0) {
3436                         DRM_ERROR("No GTT space found for object %d\n",
3437                                   reloc.target_handle);
3438                         ret = -EINVAL;
3439                         break;
3440                 }
3441
3442                 /* Validate that the target is in a valid r/w GPU domain */
3443                 if (reloc.write_domain & (reloc.write_domain - 1)) {
3444                         DRM_ERROR("reloc with multiple write domains: "
3445                                   "obj %p target %d offset %d "
3446                                   "read %08x write %08x",
3447                                   obj, reloc.target_handle,
3448                                   (int) reloc.offset,
3449                                   reloc.read_domains,
3450                                   reloc.write_domain);
3451                         ret = -EINVAL;
3452                         break;
3453                 }
3454                 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3455                     reloc.read_domains & I915_GEM_DOMAIN_CPU) {
3456                         DRM_ERROR("reloc with read/write CPU domains: "
3457                                   "obj %p target %d offset %d "
3458                                   "read %08x write %08x",
3459                                   obj, reloc.target_handle,
3460                                   (int) reloc.offset,
3461                                   reloc.read_domains,
3462                                   reloc.write_domain);
3463                         ret = -EINVAL;
3464                         break;
3465                 }
3466                 if (reloc.write_domain && target_obj->pending_write_domain &&
3467                     reloc.write_domain != target_obj->pending_write_domain) {
3468                         DRM_ERROR("Write domain conflict: "
3469                                   "obj %p target %d offset %d "
3470                                   "new %08x old %08x\n",
3471                                   obj, reloc.target_handle,
3472                                   (int) reloc.offset,
3473                                   reloc.write_domain,
3474                                   target_obj->pending_write_domain);
3475                         ret = -EINVAL;
3476                         break;
3477                 }
3478
3479                 target_obj->pending_read_domains |= reloc.read_domains;
3480                 target_obj->pending_write_domain |= reloc.write_domain;
3481
3482                 /* If the relocation already has the right value in it, no
3483                  * more work needs to be done.
3484                  */
3485                 if (target_offset == reloc.presumed_offset)
3486                         continue;
3487
3488                 /* Check that the relocation address is valid... */
3489                 if (reloc.offset > obj->base.size - 4) {
3490                         DRM_ERROR("Relocation beyond object bounds: "
3491                                   "obj %p target %d offset %d size %d.\n",
3492                                   obj, reloc.target_handle,
3493                                   (int) reloc.offset, (int) obj->base.size);
3494                         ret = -EINVAL;
3495                         break;
3496                 }
3497                 if (reloc.offset & 3) {
3498                         DRM_ERROR("Relocation not 4-byte aligned: "
3499                                   "obj %p target %d offset %d.\n",
3500                                   obj, reloc.target_handle,
3501                                   (int) reloc.offset);
3502                         ret = -EINVAL;
3503                         break;
3504                 }
3505
3506                 /* and points to somewhere within the target object. */
3507                 if (reloc.delta >= target_obj->size) {
3508                         DRM_ERROR("Relocation beyond target object bounds: "
3509                                   "obj %p target %d delta %d size %d.\n",
3510                                   obj, reloc.target_handle,
3511                                   (int) reloc.delta, (int) target_obj->size);
3512                         ret = -EINVAL;
3513                         break;
3514                 }
3515
3516                 reloc.delta += target_offset;
3517                 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3518                         uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3519                         char *vaddr;
3520
3521                         vaddr = kmap_atomic(obj->pages[reloc.offset >> PAGE_SHIFT]);
3522                         *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3523                         kunmap_atomic(vaddr);
3524                 } else {
3525                         uint32_t __iomem *reloc_entry;
3526                         void __iomem *reloc_page;
3527
3528                         ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1);
3529                         if (ret)
3530                                 break;
3531
3532                         /* Map the page containing the relocation we're going to perform.  */
3533                         reloc.offset += obj->gtt_offset;
3534                         reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3535                                                               reloc.offset & PAGE_MASK);
3536                         reloc_entry = (uint32_t __iomem *)
3537                                 (reloc_page + (reloc.offset & ~PAGE_MASK));
3538                         iowrite32(reloc.delta, reloc_entry);
3539                         io_mapping_unmap_atomic(reloc_page);
3540                 }
3541
3542                 /* and update the user's relocation entry */
3543                 reloc.presumed_offset = target_offset;
3544                 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3545                                               &reloc.presumed_offset,
3546                                               sizeof(reloc.presumed_offset))) {
3547                     ret = -EFAULT;
3548                     break;
3549                 }
3550         }
3551
3552         drm_gem_object_unreference(target_obj);
3553         return ret;
3554 }
3555
3556 static int
3557 i915_gem_execbuffer_pin(struct drm_device *dev,
3558                         struct drm_file *file,
3559                         struct drm_gem_object **object_list,
3560                         struct drm_i915_gem_exec_object2 *exec_list,
3561                         int count)
3562 {
3563         struct drm_i915_private *dev_priv = dev->dev_private;
3564         int ret, i, retry;
3565
3566         /* attempt to pin all of the buffers into the GTT */
3567         retry = 0;
3568         do {
3569                 ret = 0;
3570                 for (i = 0; i < count; i++) {
3571                         struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3572                         struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3573                         bool need_fence =
3574                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3575                                 obj->tiling_mode != I915_TILING_NONE;
3576
3577                         /* g33/pnv can't fence buffers in the unmappable part */
3578                         bool need_mappable =
3579                                 entry->relocation_count ? true : need_fence;
3580
3581                         /* Check fence reg constraints and rebind if necessary */
3582                         if (need_mappable && !obj->map_and_fenceable) {
3583                                 ret = i915_gem_object_unbind(&obj->base);
3584                                 if (ret)
3585                                         break;
3586                         }
3587
3588                         ret = i915_gem_object_pin(&obj->base,
3589                                                   entry->alignment,
3590                                                   need_mappable);
3591                         if (ret)
3592                                 break;
3593
3594                         /*
3595                          * Pre-965 chips need a fence register set up in order
3596                          * to properly handle blits to/from tiled surfaces.
3597                          */
3598                         if (need_fence) {
3599                                 ret = i915_gem_object_get_fence_reg(&obj->base, true);
3600                                 if (ret) {
3601                                         i915_gem_object_unpin(&obj->base);
3602                                         break;
3603                                 }
3604
3605                                 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3606                         }
3607
3608                         entry->offset = obj->gtt_offset;
3609                 }
3610
3611                 while (i--)
3612                         i915_gem_object_unpin(object_list[i]);
3613
3614                 if (ret != -ENOSPC || retry > 1)
3615                         return ret;
3616
3617                 /* First attempt, just clear anything that is purgeable.
3618                  * Second attempt, clear the entire GTT.
3619                  */
3620                 ret = i915_gem_evict_everything(dev, retry == 0);
3621                 if (ret)
3622                         return ret;
3623
3624                 retry++;
3625         } while (1);
3626 }
3627
3628 static int
3629 i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3630                                 struct drm_file *file,
3631                                 struct intel_ring_buffer *ring,
3632                                 struct drm_gem_object **objects,
3633                                 int count)
3634 {
3635         struct change_domains cd;
3636         int ret, i;
3637
3638         cd.invalidate_domains = 0;
3639         cd.flush_domains = 0;
3640         cd.flush_rings = 0;
3641         for (i = 0; i < count; i++)
3642                 i915_gem_object_set_to_gpu_domain(objects[i], ring, &cd);
3643
3644         if (cd.invalidate_domains | cd.flush_domains) {
3645 #if WATCH_EXEC
3646                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3647                           __func__,
3648                          cd.invalidate_domains,
3649                          cd.flush_domains);
3650 #endif
3651                 i915_gem_flush(dev, file,
3652                                cd.invalidate_domains,
3653                                cd.flush_domains,
3654                                cd.flush_rings);
3655         }
3656
3657         for (i = 0; i < count; i++) {
3658                 struct drm_i915_gem_object *obj = to_intel_bo(objects[i]);
3659                 /* XXX replace with semaphores */
3660                 if (obj->ring && ring != obj->ring) {
3661                         ret = i915_gem_object_wait_rendering(&obj->base, true);
3662                         if (ret)
3663                                 return ret;
3664                 }
3665         }
3666
3667         return 0;
3668 }
3669
3670 /* Throttle our rendering by waiting until the ring has completed our requests
3671  * emitted over 20 msec ago.
3672  *
3673  * Note that if we were to use the current jiffies each time around the loop,
3674  * we wouldn't escape the function with any frames outstanding if the time to
3675  * render a frame was over 20ms.
3676  *
3677  * This should get us reasonable parallelism between CPU and GPU but also
3678  * relatively low latency when blocking on a particular request to finish.
3679  */
3680 static int
3681 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3682 {
3683         struct drm_i915_private *dev_priv = dev->dev_private;
3684         struct drm_i915_file_private *file_priv = file->driver_priv;
3685         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3686         struct drm_i915_gem_request *request;
3687         struct intel_ring_buffer *ring = NULL;
3688         u32 seqno = 0;
3689         int ret;
3690
3691         spin_lock(&file_priv->mm.lock);
3692         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3693                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3694                         break;
3695
3696                 ring = request->ring;
3697                 seqno = request->seqno;
3698         }
3699         spin_unlock(&file_priv->mm.lock);
3700
3701         if (seqno == 0)
3702                 return 0;
3703
3704         ret = 0;
3705         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3706                 /* And wait for the seqno passing without holding any locks and
3707                  * causing extra latency for others. This is safe as the irq
3708                  * generation is designed to be run atomically and so is
3709                  * lockless.
3710                  */
3711                 ring->user_irq_get(ring);
3712                 ret = wait_event_interruptible(ring->irq_queue,
3713                                                i915_seqno_passed(ring->get_seqno(ring), seqno)
3714                                                || atomic_read(&dev_priv->mm.wedged));
3715                 ring->user_irq_put(ring);
3716
3717                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3718                         ret = -EIO;
3719         }
3720
3721         if (ret == 0)
3722                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3723
3724         return ret;
3725 }
3726
3727 static int
3728 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3729                           uint64_t exec_offset)
3730 {
3731         uint32_t exec_start, exec_len;
3732
3733         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3734         exec_len = (uint32_t) exec->batch_len;
3735
3736         if ((exec_start | exec_len) & 0x7)
3737                 return -EINVAL;
3738
3739         if (!exec_start)
3740                 return -EINVAL;
3741
3742         return 0;
3743 }
3744
3745 static int
3746 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3747                    int count)
3748 {
3749         int i;
3750
3751         for (i = 0; i < count; i++) {
3752                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3753                 int length; /* limited by fault_in_pages_readable() */
3754
3755                 /* First check for malicious input causing overflow */
3756                 if (exec[i].relocation_count >
3757                     INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
3758                         return -EINVAL;
3759
3760                 length = exec[i].relocation_count *
3761                         sizeof(struct drm_i915_gem_relocation_entry);
3762                 if (!access_ok(VERIFY_READ, ptr, length))
3763                         return -EFAULT;
3764
3765                 /* we may also need to update the presumed offsets */
3766                 if (!access_ok(VERIFY_WRITE, ptr, length))
3767                         return -EFAULT;
3768
3769                 if (fault_in_pages_readable(ptr, length))
3770                         return -EFAULT;
3771         }
3772
3773         return 0;
3774 }
3775
3776 static int
3777 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3778                        struct drm_file *file,
3779                        struct drm_i915_gem_execbuffer2 *args,
3780                        struct drm_i915_gem_exec_object2 *exec_list)
3781 {
3782         drm_i915_private_t *dev_priv = dev->dev_private;
3783         struct drm_gem_object **object_list = NULL;
3784         struct drm_gem_object *batch_obj;
3785         struct drm_clip_rect *cliprects = NULL;
3786         struct drm_i915_gem_request *request = NULL;
3787         int ret, i, flips;
3788         uint64_t exec_offset;
3789
3790         struct intel_ring_buffer *ring = NULL;
3791
3792         ret = i915_gem_check_is_wedged(dev);
3793         if (ret)
3794                 return ret;
3795
3796         ret = validate_exec_list(exec_list, args->buffer_count);
3797         if (ret)
3798                 return ret;
3799
3800 #if WATCH_EXEC
3801         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3802                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3803 #endif
3804         switch (args->flags & I915_EXEC_RING_MASK) {
3805         case I915_EXEC_DEFAULT:
3806         case I915_EXEC_RENDER:
3807                 ring = &dev_priv->render_ring;
3808                 break;
3809         case I915_EXEC_BSD:
3810                 if (!HAS_BSD(dev)) {
3811                         DRM_ERROR("execbuf with invalid ring (BSD)\n");
3812                         return -EINVAL;
3813                 }
3814                 ring = &dev_priv->bsd_ring;
3815                 break;
3816         case I915_EXEC_BLT:
3817                 if (!HAS_BLT(dev)) {
3818                         DRM_ERROR("execbuf with invalid ring (BLT)\n");
3819                         return -EINVAL;
3820                 }
3821                 ring = &dev_priv->blt_ring;
3822                 break;
3823         default:
3824                 DRM_ERROR("execbuf with unknown ring: %d\n",
3825                           (int)(args->flags & I915_EXEC_RING_MASK));
3826                 return -EINVAL;
3827         }
3828
3829         if (args->buffer_count < 1) {
3830                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3831                 return -EINVAL;
3832         }
3833         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3834         if (object_list == NULL) {
3835                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3836                           args->buffer_count);
3837                 ret = -ENOMEM;
3838                 goto pre_mutex_err;
3839         }
3840
3841         if (args->num_cliprects != 0) {
3842                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3843                                     GFP_KERNEL);
3844                 if (cliprects == NULL) {
3845                         ret = -ENOMEM;
3846                         goto pre_mutex_err;
3847                 }
3848
3849                 ret = copy_from_user(cliprects,
3850                                      (struct drm_clip_rect __user *)
3851                                      (uintptr_t) args->cliprects_ptr,
3852                                      sizeof(*cliprects) * args->num_cliprects);
3853                 if (ret != 0) {
3854                         DRM_ERROR("copy %d cliprects failed: %d\n",
3855                                   args->num_cliprects, ret);
3856                         ret = -EFAULT;
3857                         goto pre_mutex_err;
3858                 }
3859         }
3860
3861         request = kzalloc(sizeof(*request), GFP_KERNEL);
3862         if (request == NULL) {
3863                 ret = -ENOMEM;
3864                 goto pre_mutex_err;
3865         }
3866
3867         ret = i915_mutex_lock_interruptible(dev);
3868         if (ret)
3869                 goto pre_mutex_err;
3870
3871         if (dev_priv->mm.suspended) {
3872                 mutex_unlock(&dev->struct_mutex);
3873                 ret = -EBUSY;
3874                 goto pre_mutex_err;
3875         }
3876
3877         /* Look up object handles */
3878         for (i = 0; i < args->buffer_count; i++) {
3879                 struct drm_i915_gem_object *obj_priv;
3880
3881                 object_list[i] = drm_gem_object_lookup(dev, file,
3882                                                        exec_list[i].handle);
3883                 if (object_list[i] == NULL) {
3884                         DRM_ERROR("Invalid object handle %d at index %d\n",
3885                                    exec_list[i].handle, i);
3886                         /* prevent error path from reading uninitialized data */
3887                         args->buffer_count = i + 1;
3888                         ret = -ENOENT;
3889                         goto err;
3890                 }
3891
3892                 obj_priv = to_intel_bo(object_list[i]);
3893                 if (obj_priv->in_execbuffer) {
3894                         DRM_ERROR("Object %p appears more than once in object list\n",
3895                                    object_list[i]);
3896                         /* prevent error path from reading uninitialized data */
3897                         args->buffer_count = i + 1;
3898                         ret = -EINVAL;
3899                         goto err;
3900                 }
3901                 obj_priv->in_execbuffer = true;
3902         }
3903
3904         /* Move the objects en-masse into the GTT, evicting if necessary. */
3905         ret = i915_gem_execbuffer_pin(dev, file,
3906                                       object_list, exec_list,
3907                                       args->buffer_count);
3908         if (ret)
3909                 goto err;
3910
3911         /* The objects are in their final locations, apply the relocations. */
3912         for (i = 0; i < args->buffer_count; i++) {
3913                 struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]);
3914                 obj->base.pending_read_domains = 0;
3915                 obj->base.pending_write_domain = 0;
3916                 ret = i915_gem_execbuffer_relocate(obj, file, &exec_list[i]);
3917                 if (ret)
3918                         goto err;
3919         }
3920
3921         /* Set the pending read domains for the batch buffer to COMMAND */
3922         batch_obj = object_list[args->buffer_count-1];
3923         if (batch_obj->pending_write_domain) {
3924                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3925                 ret = -EINVAL;
3926                 goto err;
3927         }
3928         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3929
3930         /* Sanity check the batch buffer */
3931         exec_offset = to_intel_bo(batch_obj)->gtt_offset;
3932         ret = i915_gem_check_execbuffer(args, exec_offset);
3933         if (ret != 0) {
3934                 DRM_ERROR("execbuf with invalid offset/length\n");
3935                 goto err;
3936         }
3937
3938         ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
3939                                               object_list, args->buffer_count);
3940         if (ret)
3941                 goto err;
3942
3943 #if WATCH_COHERENCY
3944         for (i = 0; i < args->buffer_count; i++) {
3945                 i915_gem_object_check_coherency(object_list[i],
3946                                                 exec_list[i].handle);
3947         }
3948 #endif
3949
3950 #if WATCH_EXEC
3951         i915_gem_dump_object(batch_obj,
3952                               args->batch_len,
3953                               __func__,
3954                               ~0);
3955 #endif
3956
3957         /* Check for any pending flips. As we only maintain a flip queue depth
3958          * of 1, we can simply insert a WAIT for the next display flip prior
3959          * to executing the batch and avoid stalling the CPU.
3960          */
3961         flips = 0;
3962         for (i = 0; i < args->buffer_count; i++) {
3963                 if (object_list[i]->write_domain)
3964                         flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3965         }
3966         if (flips) {
3967                 int plane, flip_mask;
3968
3969                 for (plane = 0; flips >> plane; plane++) {
3970                         if (((flips >> plane) & 1) == 0)
3971                                 continue;
3972
3973                         if (plane)
3974                                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3975                         else
3976                                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3977
3978                         ret = intel_ring_begin(ring, 2);
3979                         if (ret)
3980                                 goto err;
3981
3982                         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
3983                         intel_ring_emit(ring, MI_NOOP);
3984                         intel_ring_advance(ring);
3985                 }
3986         }
3987
3988         /* Exec the batchbuffer */
3989         ret = ring->dispatch_execbuffer(ring, args, cliprects, exec_offset);
3990         if (ret) {
3991                 DRM_ERROR("dispatch failed %d\n", ret);
3992                 goto err;
3993         }
3994
3995         for (i = 0; i < args->buffer_count; i++) {
3996                 struct drm_gem_object *obj = object_list[i];
3997
3998                 obj->read_domains = obj->pending_read_domains;
3999                 obj->write_domain = obj->pending_write_domain;
4000
4001                 i915_gem_object_move_to_active(obj, ring);
4002                 if (obj->write_domain) {
4003                         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4004                         obj_priv->dirty = 1;
4005                         list_move_tail(&obj_priv->gpu_write_list,
4006                                        &ring->gpu_write_list);
4007                         intel_mark_busy(dev, obj);
4008                 }
4009
4010                 trace_i915_gem_object_change_domain(obj,
4011                                                     obj->read_domains,
4012                                                     obj->write_domain);
4013         }
4014
4015         /*
4016          * Ensure that the commands in the batch buffer are
4017          * finished before the interrupt fires
4018          */
4019         i915_retire_commands(dev, ring);
4020
4021         if (i915_add_request(dev, file, request, ring))
4022                 i915_gem_next_request_seqno(dev, ring);
4023         else
4024                 request = NULL;
4025
4026 err:
4027         for (i = 0; i < args->buffer_count; i++) {
4028                 if (object_list[i] == NULL)
4029                     break;
4030
4031                 to_intel_bo(object_list[i])->in_execbuffer = false;
4032                 drm_gem_object_unreference(object_list[i]);
4033         }
4034
4035         mutex_unlock(&dev->struct_mutex);
4036
4037 pre_mutex_err:
4038         drm_free_large(object_list);
4039         kfree(cliprects);
4040         kfree(request);
4041
4042         return ret;
4043 }
4044
4045 /*
4046  * Legacy execbuffer just creates an exec2 list from the original exec object
4047  * list array and passes it to the real function.
4048  */
4049 int
4050 i915_gem_execbuffer(struct drm_device *dev, void *data,
4051                     struct drm_file *file_priv)
4052 {
4053         struct drm_i915_gem_execbuffer *args = data;
4054         struct drm_i915_gem_execbuffer2 exec2;
4055         struct drm_i915_gem_exec_object *exec_list = NULL;
4056         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4057         int ret, i;
4058
4059 #if WATCH_EXEC
4060         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4061                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4062 #endif
4063
4064         if (args->buffer_count < 1) {
4065                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4066                 return -EINVAL;
4067         }
4068
4069         /* Copy in the exec list from userland */
4070         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4071         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4072         if (exec_list == NULL || exec2_list == NULL) {
4073                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4074                           args->buffer_count);
4075                 drm_free_large(exec_list);
4076                 drm_free_large(exec2_list);
4077                 return -ENOMEM;
4078         }
4079         ret = copy_from_user(exec_list,
4080                              (struct drm_i915_relocation_entry __user *)
4081                              (uintptr_t) args->buffers_ptr,
4082                              sizeof(*exec_list) * args->buffer_count);
4083         if (ret != 0) {
4084                 DRM_ERROR("copy %d exec entries failed %d\n",
4085                           args->buffer_count, ret);
4086                 drm_free_large(exec_list);
4087                 drm_free_large(exec2_list);
4088                 return -EFAULT;
4089         }
4090
4091         for (i = 0; i < args->buffer_count; i++) {
4092                 exec2_list[i].handle = exec_list[i].handle;
4093                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4094                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4095                 exec2_list[i].alignment = exec_list[i].alignment;
4096                 exec2_list[i].offset = exec_list[i].offset;
4097                 if (INTEL_INFO(dev)->gen < 4)
4098                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4099                 else
4100                         exec2_list[i].flags = 0;
4101         }
4102
4103         exec2.buffers_ptr = args->buffers_ptr;
4104         exec2.buffer_count = args->buffer_count;
4105         exec2.batch_start_offset = args->batch_start_offset;
4106         exec2.batch_len = args->batch_len;
4107         exec2.DR1 = args->DR1;
4108         exec2.DR4 = args->DR4;
4109         exec2.num_cliprects = args->num_cliprects;
4110         exec2.cliprects_ptr = args->cliprects_ptr;
4111         exec2.flags = I915_EXEC_RENDER;
4112
4113         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4114         if (!ret) {
4115                 /* Copy the new buffer offsets back to the user's exec list. */
4116                 for (i = 0; i < args->buffer_count; i++)
4117                         exec_list[i].offset = exec2_list[i].offset;
4118                 /* ... and back out to userspace */
4119                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4120                                    (uintptr_t) args->buffers_ptr,
4121                                    exec_list,
4122                                    sizeof(*exec_list) * args->buffer_count);
4123                 if (ret) {
4124                         ret = -EFAULT;
4125                         DRM_ERROR("failed to copy %d exec entries "
4126                                   "back to user (%d)\n",
4127                                   args->buffer_count, ret);
4128                 }
4129         }
4130
4131         drm_free_large(exec_list);
4132         drm_free_large(exec2_list);
4133         return ret;
4134 }
4135
4136 int
4137 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4138                      struct drm_file *file_priv)
4139 {
4140         struct drm_i915_gem_execbuffer2 *args = data;
4141         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4142         int ret;
4143
4144 #if WATCH_EXEC
4145         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4146                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4147 #endif
4148
4149         if (args->buffer_count < 1) {
4150                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4151                 return -EINVAL;
4152         }
4153
4154         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4155         if (exec2_list == NULL) {
4156                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4157                           args->buffer_count);
4158                 return -ENOMEM;
4159         }
4160         ret = copy_from_user(exec2_list,
4161                              (struct drm_i915_relocation_entry __user *)
4162                              (uintptr_t) args->buffers_ptr,
4163                              sizeof(*exec2_list) * args->buffer_count);
4164         if (ret != 0) {
4165                 DRM_ERROR("copy %d exec entries failed %d\n",
4166                           args->buffer_count, ret);
4167                 drm_free_large(exec2_list);
4168                 return -EFAULT;
4169         }
4170
4171         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4172         if (!ret) {
4173                 /* Copy the new buffer offsets back to the user's exec list. */
4174                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4175                                    (uintptr_t) args->buffers_ptr,
4176                                    exec2_list,
4177                                    sizeof(*exec2_list) * args->buffer_count);
4178                 if (ret) {
4179                         ret = -EFAULT;
4180                         DRM_ERROR("failed to copy %d exec entries "
4181                                   "back to user (%d)\n",
4182                                   args->buffer_count, ret);
4183                 }
4184         }
4185
4186         drm_free_large(exec2_list);
4187         return ret;
4188 }
4189
4190 int
4191 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment,
4192                     bool map_and_fenceable)
4193 {
4194         struct drm_device *dev = obj->dev;
4195         struct drm_i915_private *dev_priv = dev->dev_private;
4196         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4197         int ret;
4198
4199         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4200         BUG_ON(map_and_fenceable && !map_and_fenceable);
4201         WARN_ON(i915_verify_lists(dev));
4202
4203         if (obj_priv->gtt_space != NULL) {
4204                 if ((alignment && obj_priv->gtt_offset & (alignment - 1)) ||
4205                     (map_and_fenceable && !obj_priv->map_and_fenceable)) {
4206                         WARN(obj_priv->pin_count,
4207                              "bo is already pinned with incorrect alignment:"
4208                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
4209                              " obj->map_and_fenceable=%d\n",
4210                              obj_priv->gtt_offset, alignment,
4211                              map_and_fenceable,
4212                              obj_priv->map_and_fenceable);
4213                         ret = i915_gem_object_unbind(obj);
4214                         if (ret)
4215                                 return ret;
4216                 }
4217         }
4218
4219         if (obj_priv->gtt_space == NULL) {
4220                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
4221                                                   map_and_fenceable);
4222                 if (ret)
4223                         return ret;
4224         }
4225
4226         if (obj_priv->pin_count++ == 0) {
4227                 i915_gem_info_add_pin(dev_priv, obj_priv, map_and_fenceable);
4228                 if (!obj_priv->active)
4229                         list_move_tail(&obj_priv->mm_list,
4230                                        &dev_priv->mm.pinned_list);
4231         }
4232         BUG_ON(!obj_priv->pin_mappable && map_and_fenceable);
4233
4234         WARN_ON(i915_verify_lists(dev));
4235         return 0;
4236 }
4237
4238 void
4239 i915_gem_object_unpin(struct drm_gem_object *obj)
4240 {
4241         struct drm_device *dev = obj->dev;
4242         drm_i915_private_t *dev_priv = dev->dev_private;
4243         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4244
4245         WARN_ON(i915_verify_lists(dev));
4246         BUG_ON(obj_priv->pin_count == 0);
4247         BUG_ON(obj_priv->gtt_space == NULL);
4248
4249         if (--obj_priv->pin_count == 0) {
4250                 if (!obj_priv->active)
4251                         list_move_tail(&obj_priv->mm_list,
4252                                        &dev_priv->mm.inactive_list);
4253                 i915_gem_info_remove_pin(dev_priv, obj_priv);
4254         }
4255         WARN_ON(i915_verify_lists(dev));
4256 }
4257
4258 int
4259 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4260                    struct drm_file *file_priv)
4261 {
4262         struct drm_i915_gem_pin *args = data;
4263         struct drm_gem_object *obj;
4264         struct drm_i915_gem_object *obj_priv;
4265         int ret;
4266
4267         ret = i915_mutex_lock_interruptible(dev);
4268         if (ret)
4269                 return ret;
4270
4271         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4272         if (obj == NULL) {
4273                 ret = -ENOENT;
4274                 goto unlock;
4275         }
4276         obj_priv = to_intel_bo(obj);
4277
4278         if (obj_priv->madv != I915_MADV_WILLNEED) {
4279                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4280                 ret = -EINVAL;
4281                 goto out;
4282         }
4283
4284         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4285                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4286                           args->handle);
4287                 ret = -EINVAL;
4288                 goto out;
4289         }
4290
4291         obj_priv->user_pin_count++;
4292         obj_priv->pin_filp = file_priv;
4293         if (obj_priv->user_pin_count == 1) {
4294                 ret = i915_gem_object_pin(obj, args->alignment, true);
4295                 if (ret)
4296                         goto out;
4297         }
4298
4299         /* XXX - flush the CPU caches for pinned objects
4300          * as the X server doesn't manage domains yet
4301          */
4302         i915_gem_object_flush_cpu_write_domain(obj);
4303         args->offset = obj_priv->gtt_offset;
4304 out:
4305         drm_gem_object_unreference(obj);
4306 unlock:
4307         mutex_unlock(&dev->struct_mutex);
4308         return ret;
4309 }
4310
4311 int
4312 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4313                      struct drm_file *file_priv)
4314 {
4315         struct drm_i915_gem_pin *args = data;
4316         struct drm_gem_object *obj;
4317         struct drm_i915_gem_object *obj_priv;
4318         int ret;
4319
4320         ret = i915_mutex_lock_interruptible(dev);
4321         if (ret)
4322                 return ret;
4323
4324         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4325         if (obj == NULL) {
4326                 ret = -ENOENT;
4327                 goto unlock;
4328         }
4329         obj_priv = to_intel_bo(obj);
4330
4331         if (obj_priv->pin_filp != file_priv) {
4332                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4333                           args->handle);
4334                 ret = -EINVAL;
4335                 goto out;
4336         }
4337         obj_priv->user_pin_count--;
4338         if (obj_priv->user_pin_count == 0) {
4339                 obj_priv->pin_filp = NULL;
4340                 i915_gem_object_unpin(obj);
4341         }
4342
4343 out:
4344         drm_gem_object_unreference(obj);
4345 unlock:
4346         mutex_unlock(&dev->struct_mutex);
4347         return ret;
4348 }
4349
4350 int
4351 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4352                     struct drm_file *file_priv)
4353 {
4354         struct drm_i915_gem_busy *args = data;
4355         struct drm_gem_object *obj;
4356         struct drm_i915_gem_object *obj_priv;
4357         int ret;
4358
4359         ret = i915_mutex_lock_interruptible(dev);
4360         if (ret)
4361                 return ret;
4362
4363         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4364         if (obj == NULL) {
4365                 ret = -ENOENT;
4366                 goto unlock;
4367         }
4368         obj_priv = to_intel_bo(obj);
4369
4370         /* Count all active objects as busy, even if they are currently not used
4371          * by the gpu. Users of this interface expect objects to eventually
4372          * become non-busy without any further actions, therefore emit any
4373          * necessary flushes here.
4374          */
4375         args->busy = obj_priv->active;
4376         if (args->busy) {
4377                 /* Unconditionally flush objects, even when the gpu still uses this
4378                  * object. Userspace calling this function indicates that it wants to
4379                  * use this buffer rather sooner than later, so issuing the required
4380                  * flush earlier is beneficial.
4381                  */
4382                 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4383                         i915_gem_flush_ring(dev, file_priv,
4384                                             obj_priv->ring,
4385                                             0, obj->write_domain);
4386
4387                 /* Update the active list for the hardware's current position.
4388                  * Otherwise this only updates on a delayed timer or when irqs
4389                  * are actually unmasked, and our working set ends up being
4390                  * larger than required.
4391                  */
4392                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4393
4394                 args->busy = obj_priv->active;
4395         }
4396
4397         drm_gem_object_unreference(obj);
4398 unlock:
4399         mutex_unlock(&dev->struct_mutex);
4400         return ret;
4401 }
4402
4403 int
4404 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4405                         struct drm_file *file_priv)
4406 {
4407     return i915_gem_ring_throttle(dev, file_priv);
4408 }
4409
4410 int
4411 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4412                        struct drm_file *file_priv)
4413 {
4414         struct drm_i915_gem_madvise *args = data;
4415         struct drm_gem_object *obj;
4416         struct drm_i915_gem_object *obj_priv;
4417         int ret;
4418
4419         switch (args->madv) {
4420         case I915_MADV_DONTNEED:
4421         case I915_MADV_WILLNEED:
4422             break;
4423         default:
4424             return -EINVAL;
4425         }
4426
4427         ret = i915_mutex_lock_interruptible(dev);
4428         if (ret)
4429                 return ret;
4430
4431         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4432         if (obj == NULL) {
4433                 ret = -ENOENT;
4434                 goto unlock;
4435         }
4436         obj_priv = to_intel_bo(obj);
4437
4438         if (obj_priv->pin_count) {
4439                 ret = -EINVAL;
4440                 goto out;
4441         }
4442
4443         if (obj_priv->madv != __I915_MADV_PURGED)
4444                 obj_priv->madv = args->madv;
4445
4446         /* if the object is no longer bound, discard its backing storage */
4447         if (i915_gem_object_is_purgeable(obj_priv) &&
4448             obj_priv->gtt_space == NULL)
4449                 i915_gem_object_truncate(obj);
4450
4451         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4452
4453 out:
4454         drm_gem_object_unreference(obj);
4455 unlock:
4456         mutex_unlock(&dev->struct_mutex);
4457         return ret;
4458 }
4459
4460 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4461                                               size_t size)
4462 {
4463         struct drm_i915_private *dev_priv = dev->dev_private;
4464         struct drm_i915_gem_object *obj;
4465
4466         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4467         if (obj == NULL)
4468                 return NULL;
4469
4470         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4471                 kfree(obj);
4472                 return NULL;
4473         }
4474
4475         i915_gem_info_add_obj(dev_priv, size);
4476
4477         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4478         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4479
4480         obj->agp_type = AGP_USER_MEMORY;
4481         obj->base.driver_private = NULL;
4482         obj->fence_reg = I915_FENCE_REG_NONE;
4483         INIT_LIST_HEAD(&obj->mm_list);
4484         INIT_LIST_HEAD(&obj->ring_list);
4485         INIT_LIST_HEAD(&obj->gpu_write_list);
4486         obj->madv = I915_MADV_WILLNEED;
4487         /* Avoid an unnecessary call to unbind on the first bind. */
4488         obj->map_and_fenceable = true;
4489
4490         return &obj->base;
4491 }
4492
4493 int i915_gem_init_object(struct drm_gem_object *obj)
4494 {
4495         BUG();
4496
4497         return 0;
4498 }
4499
4500 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4501 {
4502         struct drm_device *dev = obj->dev;
4503         drm_i915_private_t *dev_priv = dev->dev_private;
4504         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4505         int ret;
4506
4507         ret = i915_gem_object_unbind(obj);
4508         if (ret == -ERESTARTSYS) {
4509                 list_move(&obj_priv->mm_list,
4510                           &dev_priv->mm.deferred_free_list);
4511                 return;
4512         }
4513
4514         if (obj->map_list.map)
4515                 i915_gem_free_mmap_offset(obj);
4516
4517         drm_gem_object_release(obj);
4518         i915_gem_info_remove_obj(dev_priv, obj->size);
4519
4520         kfree(obj_priv->page_cpu_valid);
4521         kfree(obj_priv->bit_17);
4522         kfree(obj_priv);
4523 }
4524
4525 void i915_gem_free_object(struct drm_gem_object *obj)
4526 {
4527         struct drm_device *dev = obj->dev;
4528         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4529
4530         trace_i915_gem_object_destroy(obj);
4531
4532         while (obj_priv->pin_count > 0)
4533                 i915_gem_object_unpin(obj);
4534
4535         if (obj_priv->phys_obj)
4536                 i915_gem_detach_phys_object(dev, obj);
4537
4538         i915_gem_free_object_tail(obj);
4539 }
4540
4541 int
4542 i915_gem_idle(struct drm_device *dev)
4543 {
4544         drm_i915_private_t *dev_priv = dev->dev_private;
4545         int ret;
4546
4547         mutex_lock(&dev->struct_mutex);
4548
4549         if (dev_priv->mm.suspended) {
4550                 mutex_unlock(&dev->struct_mutex);
4551                 return 0;
4552         }
4553
4554         ret = i915_gpu_idle(dev);
4555         if (ret) {
4556                 mutex_unlock(&dev->struct_mutex);
4557                 return ret;
4558         }
4559
4560         /* Under UMS, be paranoid and evict. */
4561         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4562                 ret = i915_gem_evict_inactive(dev, false);
4563                 if (ret) {
4564                         mutex_unlock(&dev->struct_mutex);
4565                         return ret;
4566                 }
4567         }
4568
4569         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4570          * We need to replace this with a semaphore, or something.
4571          * And not confound mm.suspended!
4572          */
4573         dev_priv->mm.suspended = 1;
4574         del_timer_sync(&dev_priv->hangcheck_timer);
4575
4576         i915_kernel_lost_context(dev);
4577         i915_gem_cleanup_ringbuffer(dev);
4578
4579         mutex_unlock(&dev->struct_mutex);
4580
4581         /* Cancel the retire work handler, which should be idle now. */
4582         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4583
4584         return 0;
4585 }
4586
4587 /*
4588  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4589  * over cache flushing.
4590  */
4591 static int
4592 i915_gem_init_pipe_control(struct drm_device *dev)
4593 {
4594         drm_i915_private_t *dev_priv = dev->dev_private;
4595         struct drm_gem_object *obj;
4596         struct drm_i915_gem_object *obj_priv;
4597         int ret;
4598
4599         obj = i915_gem_alloc_object(dev, 4096);
4600         if (obj == NULL) {
4601                 DRM_ERROR("Failed to allocate seqno page\n");
4602                 ret = -ENOMEM;
4603                 goto err;
4604         }
4605         obj_priv = to_intel_bo(obj);
4606         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4607
4608         ret = i915_gem_object_pin(obj, 4096, true);
4609         if (ret)
4610                 goto err_unref;
4611
4612         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4613         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4614         if (dev_priv->seqno_page == NULL)
4615                 goto err_unpin;
4616
4617         dev_priv->seqno_obj = obj;
4618         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4619
4620         return 0;
4621
4622 err_unpin:
4623         i915_gem_object_unpin(obj);
4624 err_unref:
4625         drm_gem_object_unreference(obj);
4626 err:
4627         return ret;
4628 }
4629
4630
4631 static void
4632 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4633 {
4634         drm_i915_private_t *dev_priv = dev->dev_private;
4635         struct drm_gem_object *obj;
4636         struct drm_i915_gem_object *obj_priv;
4637
4638         obj = dev_priv->seqno_obj;
4639         obj_priv = to_intel_bo(obj);
4640         kunmap(obj_priv->pages[0]);
4641         i915_gem_object_unpin(obj);
4642         drm_gem_object_unreference(obj);
4643         dev_priv->seqno_obj = NULL;
4644
4645         dev_priv->seqno_page = NULL;
4646 }
4647
4648 int
4649 i915_gem_init_ringbuffer(struct drm_device *dev)
4650 {
4651         drm_i915_private_t *dev_priv = dev->dev_private;
4652         int ret;
4653
4654         if (HAS_PIPE_CONTROL(dev)) {
4655                 ret = i915_gem_init_pipe_control(dev);
4656                 if (ret)
4657                         return ret;
4658         }
4659
4660         ret = intel_init_render_ring_buffer(dev);
4661         if (ret)
4662                 goto cleanup_pipe_control;
4663
4664         if (HAS_BSD(dev)) {
4665                 ret = intel_init_bsd_ring_buffer(dev);
4666                 if (ret)
4667                         goto cleanup_render_ring;
4668         }
4669
4670         if (HAS_BLT(dev)) {
4671                 ret = intel_init_blt_ring_buffer(dev);
4672                 if (ret)
4673                         goto cleanup_bsd_ring;
4674         }
4675
4676         dev_priv->next_seqno = 1;
4677
4678         return 0;
4679
4680 cleanup_bsd_ring:
4681         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4682 cleanup_render_ring:
4683         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4684 cleanup_pipe_control:
4685         if (HAS_PIPE_CONTROL(dev))
4686                 i915_gem_cleanup_pipe_control(dev);
4687         return ret;
4688 }
4689
4690 void
4691 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4692 {
4693         drm_i915_private_t *dev_priv = dev->dev_private;
4694
4695         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4696         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4697         intel_cleanup_ring_buffer(&dev_priv->blt_ring);
4698         if (HAS_PIPE_CONTROL(dev))
4699                 i915_gem_cleanup_pipe_control(dev);
4700 }
4701
4702 int
4703 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4704                        struct drm_file *file_priv)
4705 {
4706         drm_i915_private_t *dev_priv = dev->dev_private;
4707         int ret;
4708
4709         if (drm_core_check_feature(dev, DRIVER_MODESET))
4710                 return 0;
4711
4712         if (atomic_read(&dev_priv->mm.wedged)) {
4713                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4714                 atomic_set(&dev_priv->mm.wedged, 0);
4715         }
4716
4717         mutex_lock(&dev->struct_mutex);
4718         dev_priv->mm.suspended = 0;
4719
4720         ret = i915_gem_init_ringbuffer(dev);
4721         if (ret != 0) {
4722                 mutex_unlock(&dev->struct_mutex);
4723                 return ret;
4724         }
4725
4726         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4727         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4728         BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4729         BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4730         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4731         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4732         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4733         BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4734         BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4735         mutex_unlock(&dev->struct_mutex);
4736
4737         ret = drm_irq_install(dev);
4738         if (ret)
4739                 goto cleanup_ringbuffer;
4740
4741         return 0;
4742
4743 cleanup_ringbuffer:
4744         mutex_lock(&dev->struct_mutex);
4745         i915_gem_cleanup_ringbuffer(dev);
4746         dev_priv->mm.suspended = 1;
4747         mutex_unlock(&dev->struct_mutex);
4748
4749         return ret;
4750 }
4751
4752 int
4753 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4754                        struct drm_file *file_priv)
4755 {
4756         if (drm_core_check_feature(dev, DRIVER_MODESET))
4757                 return 0;
4758
4759         drm_irq_uninstall(dev);
4760         return i915_gem_idle(dev);
4761 }
4762
4763 void
4764 i915_gem_lastclose(struct drm_device *dev)
4765 {
4766         int ret;
4767
4768         if (drm_core_check_feature(dev, DRIVER_MODESET))
4769                 return;
4770
4771         ret = i915_gem_idle(dev);
4772         if (ret)
4773                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4774 }
4775
4776 static void
4777 init_ring_lists(struct intel_ring_buffer *ring)
4778 {
4779         INIT_LIST_HEAD(&ring->active_list);
4780         INIT_LIST_HEAD(&ring->request_list);
4781         INIT_LIST_HEAD(&ring->gpu_write_list);
4782 }
4783
4784 void
4785 i915_gem_load(struct drm_device *dev)
4786 {
4787         int i;
4788         drm_i915_private_t *dev_priv = dev->dev_private;
4789
4790         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4791         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4792         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4793         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4794         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4795         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4796         init_ring_lists(&dev_priv->render_ring);
4797         init_ring_lists(&dev_priv->bsd_ring);
4798         init_ring_lists(&dev_priv->blt_ring);
4799         for (i = 0; i < 16; i++)
4800                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4801         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4802                           i915_gem_retire_work_handler);
4803         init_completion(&dev_priv->error_completion);
4804
4805         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4806         if (IS_GEN3(dev)) {
4807                 u32 tmp = I915_READ(MI_ARB_STATE);
4808                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4809                         /* arb state is a masked write, so set bit + bit in mask */
4810                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4811                         I915_WRITE(MI_ARB_STATE, tmp);
4812                 }
4813         }
4814
4815         /* Old X drivers will take 0-2 for front, back, depth buffers */
4816         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4817                 dev_priv->fence_reg_start = 3;
4818
4819         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4820                 dev_priv->num_fence_regs = 16;
4821         else
4822                 dev_priv->num_fence_regs = 8;
4823
4824         /* Initialize fence registers to zero */
4825         switch (INTEL_INFO(dev)->gen) {
4826         case 6:
4827                 for (i = 0; i < 16; i++)
4828                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4829                 break;
4830         case 5:
4831         case 4:
4832                 for (i = 0; i < 16; i++)
4833                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4834                 break;
4835         case 3:
4836                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4837                         for (i = 0; i < 8; i++)
4838                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4839         case 2:
4840                 for (i = 0; i < 8; i++)
4841                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4842                 break;
4843         }
4844         i915_gem_detect_bit_6_swizzle(dev);
4845         init_waitqueue_head(&dev_priv->pending_flip_queue);
4846
4847         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4848         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4849         register_shrinker(&dev_priv->mm.inactive_shrinker);
4850 }
4851
4852 /*
4853  * Create a physically contiguous memory object for this object
4854  * e.g. for cursor + overlay regs
4855  */
4856 static int i915_gem_init_phys_object(struct drm_device *dev,
4857                                      int id, int size, int align)
4858 {
4859         drm_i915_private_t *dev_priv = dev->dev_private;
4860         struct drm_i915_gem_phys_object *phys_obj;
4861         int ret;
4862
4863         if (dev_priv->mm.phys_objs[id - 1] || !size)
4864                 return 0;
4865
4866         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4867         if (!phys_obj)
4868                 return -ENOMEM;
4869
4870         phys_obj->id = id;
4871
4872         phys_obj->handle = drm_pci_alloc(dev, size, align);
4873         if (!phys_obj->handle) {
4874                 ret = -ENOMEM;
4875                 goto kfree_obj;
4876         }
4877 #ifdef CONFIG_X86
4878         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4879 #endif
4880
4881         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4882
4883         return 0;
4884 kfree_obj:
4885         kfree(phys_obj);
4886         return ret;
4887 }
4888
4889 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4890 {
4891         drm_i915_private_t *dev_priv = dev->dev_private;
4892         struct drm_i915_gem_phys_object *phys_obj;
4893
4894         if (!dev_priv->mm.phys_objs[id - 1])
4895                 return;
4896
4897         phys_obj = dev_priv->mm.phys_objs[id - 1];
4898         if (phys_obj->cur_obj) {
4899                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4900         }
4901
4902 #ifdef CONFIG_X86
4903         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4904 #endif
4905         drm_pci_free(dev, phys_obj->handle);
4906         kfree(phys_obj);
4907         dev_priv->mm.phys_objs[id - 1] = NULL;
4908 }
4909
4910 void i915_gem_free_all_phys_object(struct drm_device *dev)
4911 {
4912         int i;
4913
4914         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4915                 i915_gem_free_phys_object(dev, i);
4916 }
4917
4918 void i915_gem_detach_phys_object(struct drm_device *dev,
4919                                  struct drm_gem_object *obj)
4920 {
4921         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4922         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4923         char *vaddr;
4924         int i;
4925         int page_count;
4926
4927         if (!obj_priv->phys_obj)
4928                 return;
4929         vaddr = obj_priv->phys_obj->handle->vaddr;
4930
4931         page_count = obj->size / PAGE_SIZE;
4932
4933         for (i = 0; i < page_count; i++) {
4934                 struct page *page = read_cache_page_gfp(mapping, i,
4935                                                         GFP_HIGHUSER | __GFP_RECLAIMABLE);
4936                 if (!IS_ERR(page)) {
4937                         char *dst = kmap_atomic(page);
4938                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4939                         kunmap_atomic(dst);
4940
4941                         drm_clflush_pages(&page, 1);
4942
4943                         set_page_dirty(page);
4944                         mark_page_accessed(page);
4945                         page_cache_release(page);
4946                 }
4947         }
4948         drm_agp_chipset_flush(dev);
4949
4950         obj_priv->phys_obj->cur_obj = NULL;
4951         obj_priv->phys_obj = NULL;
4952 }
4953
4954 int
4955 i915_gem_attach_phys_object(struct drm_device *dev,
4956                             struct drm_gem_object *obj,
4957                             int id,
4958                             int align)
4959 {
4960         struct address_space *mapping = obj->filp->f_path.dentry->d_inode->i_mapping;
4961         drm_i915_private_t *dev_priv = dev->dev_private;
4962         struct drm_i915_gem_object *obj_priv;
4963         int ret = 0;
4964         int page_count;
4965         int i;
4966
4967         if (id > I915_MAX_PHYS_OBJECT)
4968                 return -EINVAL;
4969
4970         obj_priv = to_intel_bo(obj);
4971
4972         if (obj_priv->phys_obj) {
4973                 if (obj_priv->phys_obj->id == id)
4974                         return 0;
4975                 i915_gem_detach_phys_object(dev, obj);
4976         }
4977
4978         /* create a new object */
4979         if (!dev_priv->mm.phys_objs[id - 1]) {
4980                 ret = i915_gem_init_phys_object(dev, id,
4981                                                 obj->size, align);
4982                 if (ret) {
4983                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4984                         return ret;
4985                 }
4986         }
4987
4988         /* bind to the object */
4989         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4990         obj_priv->phys_obj->cur_obj = obj;
4991
4992         page_count = obj->size / PAGE_SIZE;
4993
4994         for (i = 0; i < page_count; i++) {
4995                 struct page *page;
4996                 char *dst, *src;
4997
4998                 page = read_cache_page_gfp(mapping, i,
4999                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
5000                 if (IS_ERR(page))
5001                         return PTR_ERR(page);
5002
5003                 src = kmap_atomic(page);
5004                 dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
5005                 memcpy(dst, src, PAGE_SIZE);
5006                 kunmap_atomic(src);
5007
5008                 mark_page_accessed(page);
5009                 page_cache_release(page);
5010         }
5011
5012         return 0;
5013 }
5014
5015 static int
5016 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
5017                      struct drm_i915_gem_pwrite *args,
5018                      struct drm_file *file_priv)
5019 {
5020         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
5021         void *vaddr = obj_priv->phys_obj->handle->vaddr + args->offset;
5022         char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
5023
5024         DRM_DEBUG_DRIVER("vaddr %p, %lld\n", vaddr, args->size);
5025
5026         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
5027                 unsigned long unwritten;
5028
5029                 /* The physical object once assigned is fixed for the lifetime
5030                  * of the obj, so we can safely drop the lock and continue
5031                  * to access vaddr.
5032                  */
5033                 mutex_unlock(&dev->struct_mutex);
5034                 unwritten = copy_from_user(vaddr, user_data, args->size);
5035                 mutex_lock(&dev->struct_mutex);
5036                 if (unwritten)
5037                         return -EFAULT;
5038         }
5039
5040         drm_agp_chipset_flush(dev);
5041         return 0;
5042 }
5043
5044 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5045 {
5046         struct drm_i915_file_private *file_priv = file->driver_priv;
5047
5048         /* Clean up our request list when the client is going away, so that
5049          * later retire_requests won't dereference our soon-to-be-gone
5050          * file_priv.
5051          */
5052         spin_lock(&file_priv->mm.lock);
5053         while (!list_empty(&file_priv->mm.request_list)) {
5054                 struct drm_i915_gem_request *request;
5055
5056                 request = list_first_entry(&file_priv->mm.request_list,
5057                                            struct drm_i915_gem_request,
5058                                            client_list);
5059                 list_del(&request->client_list);
5060                 request->file_priv = NULL;
5061         }
5062         spin_unlock(&file_priv->mm.lock);
5063 }
5064
5065 static int
5066 i915_gpu_is_active(struct drm_device *dev)
5067 {
5068         drm_i915_private_t *dev_priv = dev->dev_private;
5069         int lists_empty;
5070
5071         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5072                       list_empty(&dev_priv->mm.active_list);
5073
5074         return !lists_empty;
5075 }
5076
5077 static int
5078 i915_gem_inactive_shrink(struct shrinker *shrinker,
5079                          int nr_to_scan,
5080                          gfp_t gfp_mask)
5081 {
5082         struct drm_i915_private *dev_priv =
5083                 container_of(shrinker,
5084                              struct drm_i915_private,
5085                              mm.inactive_shrinker);
5086         struct drm_device *dev = dev_priv->dev;
5087         struct drm_i915_gem_object *obj, *next;
5088         int cnt;
5089
5090         if (!mutex_trylock(&dev->struct_mutex))
5091                 return 0;
5092
5093         /* "fast-path" to count number of available objects */
5094         if (nr_to_scan == 0) {
5095                 cnt = 0;
5096                 list_for_each_entry(obj,
5097                                     &dev_priv->mm.inactive_list,
5098                                     mm_list)
5099                         cnt++;
5100                 mutex_unlock(&dev->struct_mutex);
5101                 return cnt / 100 * sysctl_vfs_cache_pressure;
5102         }
5103
5104 rescan:
5105         /* first scan for clean buffers */
5106         i915_gem_retire_requests(dev);
5107
5108         list_for_each_entry_safe(obj, next,
5109                                  &dev_priv->mm.inactive_list,
5110                                  mm_list) {
5111                 if (i915_gem_object_is_purgeable(obj)) {
5112                         i915_gem_object_unbind(&obj->base);
5113                         if (--nr_to_scan == 0)
5114                                 break;
5115                 }
5116         }
5117
5118         /* second pass, evict/count anything still on the inactive list */
5119         cnt = 0;
5120         list_for_each_entry_safe(obj, next,
5121                                  &dev_priv->mm.inactive_list,
5122                                  mm_list) {
5123                 if (nr_to_scan) {
5124                         i915_gem_object_unbind(&obj->base);
5125                         nr_to_scan--;
5126                 } else
5127                         cnt++;
5128         }
5129
5130         if (nr_to_scan && i915_gpu_is_active(dev)) {
5131                 /*
5132                  * We are desperate for pages, so as a last resort, wait
5133                  * for the GPU to finish and discard whatever we can.
5134                  * This has a dramatic impact to reduce the number of
5135                  * OOM-killer events whilst running the GPU aggressively.
5136                  */
5137                 if (i915_gpu_idle(dev) == 0)
5138                         goto rescan;
5139         }
5140         mutex_unlock(&dev->struct_mutex);
5141         return cnt / 100 * sysctl_vfs_cache_pressure;
5142 }