drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37
  38 struct change_domains {
  39         uint32_t invalidate_domains;
  40         uint32_t flush_domains;
  41         uint32_t flush_rings;
  42 };
  43
  44 static uint32_t i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj);
  45 static uint32_t i915_gem_get_gtt_size(struct drm_i915_gem_object *obj);
  46
  47 static int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj,
  48                                                   bool pipelined);
  49 static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  50 static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
  51 static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
  52                                              int write);
  53 static int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
  54                                                      uint64_t offset,
  55                                                      uint64_t size);
  56 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
  57 static int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  58                                           bool interruptible);
  59 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
  60                                        unsigned alignment,
  61                                        bool map_and_fenceable);
  62 static void i915_gem_clear_fence_reg(struct drm_i915_gem_object *obj);
  63 static int i915_gem_phys_pwrite(struct drm_device *dev,
  64                                 struct drm_i915_gem_object *obj,
  65                                 struct drm_i915_gem_pwrite *args,
  66                                 struct drm_file *file);
  67 static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
  68
  69 static int i915_gem_inactive_shrink(struct shrinker *shrinker,
  70                                     int nr_to_scan,
  71                                     gfp_t gfp_mask);
  72
  73
  74 /* some bookkeeping */
  75 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
  76                                   size_t size)
  77 {
  78         dev_priv->mm.object_count++;
  79         dev_priv->mm.object_memory += size;
  80 }
  81
  82 static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
  83                                      size_t size)
  84 {
  85         dev_priv->mm.object_count--;
  86         dev_priv->mm.object_memory -= size;
  87 }
  88
  89 static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
  90                                   struct drm_i915_gem_object *obj)
  91 {
  92         dev_priv->mm.gtt_count++;
  93         dev_priv->mm.gtt_memory += obj->gtt_space->size;
  94         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
  95                 dev_priv->mm.mappable_gtt_used +=
  96                         min_t(size_t, obj->gtt_space->size,
  97                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
  98         }
  99         list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
 100 }
 101
 102 static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
 103                                      struct drm_i915_gem_object *obj)
 104 {
 105         dev_priv->mm.gtt_count--;
 106         dev_priv->mm.gtt_memory -= obj->gtt_space->size;
 107         if (obj->gtt_offset < dev_priv->mm.gtt_mappable_end) {
 108                 dev_priv->mm.mappable_gtt_used -=
 109                         min_t(size_t, obj->gtt_space->size,
 110                               dev_priv->mm.gtt_mappable_end - obj->gtt_offset);
 111         }
 112         list_del_init(&obj->gtt_list);
 113 }
 114
 115 /**
 116  * Update the mappable working set counters. Call _only_ when there is a change
 117  * in one of (pin|fault)_mappable and update *_mappable _before_ calling.
 118  * @mappable: new state the changed mappable flag (either pin_ or fault_).
 119  */
 120 static void
 121 i915_gem_info_update_mappable(struct drm_i915_private *dev_priv,
 122                               struct drm_i915_gem_object *obj,
 123                               bool mappable)
 124 {
 125         if (mappable) {
 126                 if (obj->pin_mappable && obj->fault_mappable)
 127                         /* Combined state was already mappable. */
 128                         return;
 129                 dev_priv->mm.gtt_mappable_count++;
 130                 dev_priv->mm.gtt_mappable_memory += obj->gtt_space->size;
 131         } else {
 132                 if (obj->pin_mappable || obj->fault_mappable)
 133                         /* Combined state still mappable. */
 134                         return;
 135                 dev_priv->mm.gtt_mappable_count--;
 136                 dev_priv->mm.gtt_mappable_memory -= obj->gtt_space->size;
 137         }
 138 }
 139
 140 static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
 141                                   struct drm_i915_gem_object *obj,
 142                                   bool mappable)
 143 {
 144         dev_priv->mm.pin_count++;
 145         dev_priv->mm.pin_memory += obj->gtt_space->size;
 146         if (mappable) {
 147                 obj->pin_mappable = true;
 148                 i915_gem_info_update_mappable(dev_priv, obj, true);
 149         }
 150 }
 151
 152 static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
 153                                      struct drm_i915_gem_object *obj)
 154 {
 155         dev_priv->mm.pin_count--;
 156         dev_priv->mm.pin_memory -= obj->gtt_space->size;
 157         if (obj->pin_mappable) {
 158                 obj->pin_mappable = false;
 159                 i915_gem_info_update_mappable(dev_priv, obj, false);
 160         }
 161 }
 162
 163 int
 164 i915_gem_check_is_wedged(struct drm_device *dev)
 165 {
 166         struct drm_i915_private *dev_priv = dev->dev_private;
 167         struct completion *x = &dev_priv->error_completion;
 168         unsigned long flags;
 169         int ret;
 170
 171         if (!atomic_read(&dev_priv->mm.wedged))
 172                 return 0;
 173
 174         ret = wait_for_completion_interruptible(x);
 175         if (ret)
 176                 return ret;
 177
 178         /* Success, we reset the GPU! */
 179         if (!atomic_read(&dev_priv->mm.wedged))
 180                 return 0;
 181
 182         /* GPU is hung, bump the completion count to account for
 183          * the token we just consumed so that we never hit zero and
 184          * end up waiting upon a subsequent completion event that
 185          * will never happen.
 186          */
 187         spin_lock_irqsave(&x->wait.lock, flags);
 188         x->done++;
 189         spin_unlock_irqrestore(&x->wait.lock, flags);
 190         return -EIO;
 191 }
 192
 193 static int i915_mutex_lock_interruptible(struct drm_device *dev)
 194 {
 195         struct drm_i915_private *dev_priv = dev->dev_private;
 196         int ret;
 197
 198         ret = i915_gem_check_is_wedged(dev);
 199         if (ret)
 200                 return ret;
 201
 202         ret = mutex_lock_interruptible(&dev->struct_mutex);
 203         if (ret)
 204                 return ret;
 205
 206         if (atomic_read(&dev_priv->mm.wedged)) {
 207                 mutex_unlock(&dev->struct_mutex);
 208                 return -EAGAIN;
 209         }
 210
 211         WARN_ON(i915_verify_lists(dev));
 212         return 0;
 213 }
 214
 215 static inline bool
 216 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 217 {
 218         return obj->gtt_space && !obj->active && obj->pin_count == 0;
 219 }
 220
 221 int i915_gem_do_init(struct drm_device *dev,
 222                      unsigned long start,
 223                      unsigned long mappable_end,
 224                      unsigned long end)
 225 {
 226         drm_i915_private_t *dev_priv = dev->dev_private;
 227
 228         if (start >= end ||
 229             (start & (PAGE_SIZE - 1)) != 0 ||
 230             (end & (PAGE_SIZE - 1)) != 0) {
 231                 return -EINVAL;
 232         }
 233
 234         drm_mm_init(&dev_priv->mm.gtt_space, start,
 235                     end - start);
 236
 237         dev_priv->mm.gtt_total = end - start;
 238         dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
 239         dev_priv->mm.gtt_mappable_end = mappable_end;
 240
 241         return 0;
 242 }
 243
 244 int
 245 i915_gem_init_ioctl(struct drm_device *dev, void *data,
 246                     struct drm_file *file)
 247 {
 248         struct drm_i915_gem_init *args = data;
 249         int ret;
 250
 251         mutex_lock(&dev->struct_mutex);
 252         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
 253         mutex_unlock(&dev->struct_mutex);
 254
 255         return ret;
 256 }
 257
 258 int
 259 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 260                             struct drm_file *file)
 261 {
 262         struct drm_i915_private *dev_priv = dev->dev_private;
 263         struct drm_i915_gem_get_aperture *args = data;
 264
 265         if (!(dev->driver->driver_features & DRIVER_GEM))
 266                 return -ENODEV;
 267
 268         mutex_lock(&dev->struct_mutex);
 269         args->aper_size = dev_priv->mm.gtt_total;
 270         args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
 271         mutex_unlock(&dev->struct_mutex);
 272
 273         return 0;
 274 }
 275
 276
 277 /**
 278  * Creates a new mm object and returns a handle to it.
 279  */
 280 int
 281 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 282                       struct drm_file *file)
 283 {
 284         struct drm_i915_gem_create *args = data;
 285         struct drm_i915_gem_object *obj;
 286         int ret;
 287         u32 handle;
 288
 289         args->size = roundup(args->size, PAGE_SIZE);
 290
 291         /* Allocate the new object */
 292         obj = i915_gem_alloc_object(dev, args->size);
 293         if (obj == NULL)
 294                 return -ENOMEM;
 295
 296         ret = drm_gem_handle_create(file, &obj->base, &handle);
 297         if (ret) {
 298                 drm_gem_object_release(&obj->base);
 299                 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
 300                 kfree(obj);
 301                 return ret;
 302         }
 303
 304         /* drop reference from allocate - handle holds it now */
 305         drm_gem_object_unreference(&obj->base);
 306         trace_i915_gem_object_create(obj);
 307
 308         args->handle = handle;
 309         return 0;
 310 }
 311
 312 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 313 {
 314         drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
 315
 316         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 317                 obj->tiling_mode != I915_TILING_NONE;
 318 }
 319
 320 static inline void
 321 slow_shmem_copy(struct page *dst_page,
 322                 int dst_offset,
 323                 struct page *src_page,
 324                 int src_offset,
 325                 int length)
 326 {
 327         char *dst_vaddr, *src_vaddr;
 328
 329         dst_vaddr = kmap(dst_page);
 330         src_vaddr = kmap(src_page);
 331
 332         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 333
 334         kunmap(src_page);
 335         kunmap(dst_page);
 336 }
 337
 338 static inline void
 339 slow_shmem_bit17_copy(struct page *gpu_page,
 340                       int gpu_offset,
 341                       struct page *cpu_page,
 342                       int cpu_offset,
 343                       int length,
 344                       int is_read)
 345 {
 346         char *gpu_vaddr, *cpu_vaddr;
 347
 348         /* Use the unswizzled path if this page isn't affected. */
 349         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 350                 if (is_read)
 351                         return slow_shmem_copy(cpu_page, cpu_offset,
 352                                                gpu_page, gpu_offset, length);
 353                 else
 354                         return slow_shmem_copy(gpu_page, gpu_offset,
 355                                                cpu_page, cpu_offset, length);
 356         }
 357
 358         gpu_vaddr = kmap(gpu_page);
 359         cpu_vaddr = kmap(cpu_page);
 360
 361         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 362          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 363          */
 364         while (length > 0) {
 365                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 366                 int this_length = min(cacheline_end - gpu_offset, length);
 367                 int swizzled_gpu_offset = gpu_offset ^ 64;
 368
 369                 if (is_read) {
 370                         memcpy(cpu_vaddr + cpu_offset,
 371                                gpu_vaddr + swizzled_gpu_offset,
 372                                this_length);
 373                 } else {
 374                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 375                                cpu_vaddr + cpu_offset,
 376                                this_length);
 377                 }
 378                 cpu_offset += this_length;
 379                 gpu_offset += this_length;
 380                 length -= this_length;
 381         }
 382
 383         kunmap(cpu_page);
 384         kunmap(gpu_page);
 385 }
 386
 387 /**
 388  * This is the fast shmem pread path, which attempts to copy_from_user directly
 389  * from the backing pages of the object to the user's address space.  On a
 390  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 391  */
 392 static int
 393 i915_gem_shmem_pread_fast(struct drm_device *dev,
 394                           struct drm_i915_gem_object *obj,
 395                           struct drm_i915_gem_pread *args,
 396                           struct drm_file *file)
 397 {
 398         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 399         ssize_t remain;
 400         loff_t offset;
 401         char __user *user_data;
 402         int page_offset, page_length;
 403
 404         user_data = (char __user *) (uintptr_t) args->data_ptr;
 405         remain = args->size;
 406
 407         offset = args->offset;
 408
 409         while (remain > 0) {
 410                 struct page *page;
 411                 char *vaddr;
 412                 int ret;
 413
 414                 /* Operation in this page
 415                  *
 416                  * page_offset = offset within page
 417                  * page_length = bytes to copy for this page
 418                  */
 419                 page_offset = offset & (PAGE_SIZE-1);
 420                 page_length = remain;
 421                 if ((page_offset + remain) > PAGE_SIZE)
 422                         page_length = PAGE_SIZE - page_offset;
 423
 424                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 425                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 426                 if (IS_ERR(page))
 427                         return PTR_ERR(page);
 428
 429                 vaddr = kmap_atomic(page);
 430                 ret = __copy_to_user_inatomic(user_data,
 431                                               vaddr + page_offset,
 432                                               page_length);
 433                 kunmap_atomic(vaddr);
 434
 435                 mark_page_accessed(page);
 436                 page_cache_release(page);
 437                 if (ret)
 438                         return -EFAULT;
 439
 440                 remain -= page_length;
 441                 user_data += page_length;
 442                 offset += page_length;
 443         }
 444
 445         return 0;
 446 }
 447
 448 /**
 449  * This is the fallback shmem pread path, which allocates temporary storage
 450  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 451  * can copy out of the object's backing pages while holding the struct mutex
 452  * and not take page faults.
 453  */
 454 static int
 455 i915_gem_shmem_pread_slow(struct drm_device *dev,
 456                           struct drm_i915_gem_object *obj,
 457                           struct drm_i915_gem_pread *args,
 458                           struct drm_file *file)
 459 {
 460         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 461         struct mm_struct *mm = current->mm;
 462         struct page **user_pages;
 463         ssize_t remain;
 464         loff_t offset, pinned_pages, i;
 465         loff_t first_data_page, last_data_page, num_pages;
 466         int shmem_page_offset;
 467         int data_page_index, data_page_offset;
 468         int page_length;
 469         int ret;
 470         uint64_t data_ptr = args->data_ptr;
 471         int do_bit17_swizzling;
 472
 473         remain = args->size;
 474
 475         /* Pin the user pages containing the data.  We can't fault while
 476          * holding the struct mutex, yet we want to hold it while
 477          * dereferencing the user data.
 478          */
 479         first_data_page = data_ptr / PAGE_SIZE;
 480         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 481         num_pages = last_data_page - first_data_page + 1;
 482
 483         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 484         if (user_pages == NULL)
 485                 return -ENOMEM;
 486
 487         mutex_unlock(&dev->struct_mutex);
 488         down_read(&mm->mmap_sem);
 489         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 490                                       num_pages, 1, 0, user_pages, NULL);
 491         up_read(&mm->mmap_sem);
 492         mutex_lock(&dev->struct_mutex);
 493         if (pinned_pages < num_pages) {
 494                 ret = -EFAULT;
 495                 goto out;
 496         }
 497
 498         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 499                                                         args->offset,
 500                                                         args->size);
 501         if (ret)
 502                 goto out;
 503
 504         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 505
 506         offset = args->offset;
 507
 508         while (remain > 0) {
 509                 struct page *page;
 510
 511                 /* Operation in this page
 512                  *
 513                  * shmem_page_offset = offset within page in shmem file
 514                  * data_page_index = page number in get_user_pages return
 515                  * data_page_offset = offset with data_page_index page.
 516                  * page_length = bytes to copy for this page
 517                  */
 518                 shmem_page_offset = offset & ~PAGE_MASK;
 519                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 520                 data_page_offset = data_ptr & ~PAGE_MASK;
 521
 522                 page_length = remain;
 523                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 524                         page_length = PAGE_SIZE - shmem_page_offset;
 525                 if ((data_page_offset + page_length) > PAGE_SIZE)
 526                         page_length = PAGE_SIZE - data_page_offset;
 527
 528                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 529                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 530                 if (IS_ERR(page))
 531                         return PTR_ERR(page);
 532
 533                 if (do_bit17_swizzling) {
 534                         slow_shmem_bit17_copy(page,
 535                                               shmem_page_offset,
 536                                               user_pages[data_page_index],
 537                                               data_page_offset,
 538                                               page_length,
 539                                               1);
 540                 } else {
 541                         slow_shmem_copy(user_pages[data_page_index],
 542                                         data_page_offset,
 543                                         page,
 544                                         shmem_page_offset,
 545                                         page_length);
 546                 }
 547
 548                 mark_page_accessed(page);
 549                 page_cache_release(page);
 550
 551                 remain -= page_length;
 552                 data_ptr += page_length;
 553                 offset += page_length;
 554         }
 555
 556 out:
 557         for (i = 0; i < pinned_pages; i++) {
 558                 SetPageDirty(user_pages[i]);
 559                 mark_page_accessed(user_pages[i]);
 560                 page_cache_release(user_pages[i]);
 561         }
 562         drm_free_large(user_pages);
 563
 564         return ret;
 565 }
 566
 567 /**
 568  * Reads data from the object referenced by handle.
 569  *
 570  * On error, the contents of *data are undefined.
 571  */
 572 int
 573 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 574                      struct drm_file *file)
 575 {
 576         struct drm_i915_gem_pread *args = data;
 577         struct drm_i915_gem_object *obj;
 578         int ret = 0;
 579
 580         if (args->size == 0)
 581                 return 0;
 582
 583         if (!access_ok(VERIFY_WRITE,
 584                        (char __user *)(uintptr_t)args->data_ptr,
 585                        args->size))
 586                 return -EFAULT;
 587
 588         ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
 589                                        args->size);
 590         if (ret)
 591                 return -EFAULT;
 592
 593         ret = i915_mutex_lock_interruptible(dev);
 594         if (ret)
 595                 return ret;
 596
 597         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 598         if (obj == NULL) {
 599                 ret = -ENOENT;
 600                 goto unlock;
 601         }
 602
 603         /* Bounds check source.  */
 604         if (args->offset > obj->base.size ||
 605             args->size > obj->base.size - args->offset) {
 606                 ret = -EINVAL;
 607                 goto out;
 608         }
 609
 610         ret = i915_gem_object_set_cpu_read_domain_range(obj,
 611                                                         args->offset,
 612                                                         args->size);
 613         if (ret)
 614                 goto out;
 615
 616         ret = -EFAULT;
 617         if (!i915_gem_object_needs_bit17_swizzle(obj))
 618                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
 619         if (ret == -EFAULT)
 620                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
 621
 622 out:
 623         drm_gem_object_unreference(&obj->base);
 624 unlock:
 625         mutex_unlock(&dev->struct_mutex);
 626         return ret;
 627 }
 628
 629 /* This is the fast write path which cannot handle
 630  * page faults in the source data
 631  */
 632
 633 static inline int
 634 fast_user_write(struct io_mapping *mapping,
 635                 loff_t page_base, int page_offset,
 636                 char __user *user_data,
 637                 int length)
 638 {
 639         char *vaddr_atomic;
 640         unsigned long unwritten;
 641
 642         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 643         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 644                                                       user_data, length);
 645         io_mapping_unmap_atomic(vaddr_atomic);
 646         return unwritten;
 647 }
 648
 649 /* Here's the write path which can sleep for
 650  * page faults
 651  */
 652
 653 static inline void
 654 slow_kernel_write(struct io_mapping *mapping,
 655                   loff_t gtt_base, int gtt_offset,
 656                   struct page *user_page, int user_offset,
 657                   int length)
 658 {
 659         char __iomem *dst_vaddr;
 660         char *src_vaddr;
 661
 662         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 663         src_vaddr = kmap(user_page);
 664
 665         memcpy_toio(dst_vaddr + gtt_offset,
 666                     src_vaddr + user_offset,
 667                     length);
 668
 669         kunmap(user_page);
 670         io_mapping_unmap(dst_vaddr);
 671 }
 672
 673 /**
 674  * This is the fast pwrite path, where we copy the data directly from the
 675  * user into the GTT, uncached.
 676  */
 677 static int
 678 i915_gem_gtt_pwrite_fast(struct drm_device *dev,
 679                          struct drm_i915_gem_object *obj,
 680                          struct drm_i915_gem_pwrite *args,
 681                          struct drm_file *file)
 682 {
 683         drm_i915_private_t *dev_priv = dev->dev_private;
 684         ssize_t remain;
 685         loff_t offset, page_base;
 686         char __user *user_data;
 687         int page_offset, page_length;
 688
 689         user_data = (char __user *) (uintptr_t) args->data_ptr;
 690         remain = args->size;
 691
 692         offset = obj->gtt_offset + args->offset;
 693
 694         while (remain > 0) {
 695                 /* Operation in this page
 696                  *
 697                  * page_base = page offset within aperture
 698                  * page_offset = offset within page
 699                  * page_length = bytes to copy for this page
 700                  */
 701                 page_base = (offset & ~(PAGE_SIZE-1));
 702                 page_offset = offset & (PAGE_SIZE-1);
 703                 page_length = remain;
 704                 if ((page_offset + remain) > PAGE_SIZE)
 705                         page_length = PAGE_SIZE - page_offset;
 706
 707                 /* If we get a fault while copying data, then (presumably) our
 708                  * source page isn't available.  Return the error and we'll
 709                  * retry in the slow path.
 710                  */
 711                 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
 712                                     page_offset, user_data, page_length))
 713
 714                         return -EFAULT;
 715
 716                 remain -= page_length;
 717                 user_data += page_length;
 718                 offset += page_length;
 719         }
 720
 721         return 0;
 722 }
 723
 724 /**
 725  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 726  * the memory and maps it using kmap_atomic for copying.
 727  *
 728  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 729  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 730  */
 731 static int
 732 i915_gem_gtt_pwrite_slow(struct drm_device *dev,
 733                          struct drm_i915_gem_object *obj,
 734                          struct drm_i915_gem_pwrite *args,
 735                          struct drm_file *file)
 736 {
 737         drm_i915_private_t *dev_priv = dev->dev_private;
 738         ssize_t remain;
 739         loff_t gtt_page_base, offset;
 740         loff_t first_data_page, last_data_page, num_pages;
 741         loff_t pinned_pages, i;
 742         struct page **user_pages;
 743         struct mm_struct *mm = current->mm;
 744         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 745         int ret;
 746         uint64_t data_ptr = args->data_ptr;
 747
 748         remain = args->size;
 749
 750         /* Pin the user pages containing the data.  We can't fault while
 751          * holding the struct mutex, and all of the pwrite implementations
 752          * want to hold it while dereferencing the user data.
 753          */
 754         first_data_page = data_ptr / PAGE_SIZE;
 755         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 756         num_pages = last_data_page - first_data_page + 1;
 757
 758         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 759         if (user_pages == NULL)
 760                 return -ENOMEM;
 761
 762         mutex_unlock(&dev->struct_mutex);
 763         down_read(&mm->mmap_sem);
 764         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 765                                       num_pages, 0, 0, user_pages, NULL);
 766         up_read(&mm->mmap_sem);
 767         mutex_lock(&dev->struct_mutex);
 768         if (pinned_pages < num_pages) {
 769                 ret = -EFAULT;
 770                 goto out_unpin_pages;
 771         }
 772
 773         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 774         if (ret)
 775                 goto out_unpin_pages;
 776
 777         offset = obj->gtt_offset + args->offset;
 778
 779         while (remain > 0) {
 780                 /* Operation in this page
 781                  *
 782                  * gtt_page_base = page offset within aperture
 783                  * gtt_page_offset = offset within page in aperture
 784                  * data_page_index = page number in get_user_pages return
 785                  * data_page_offset = offset with data_page_index page.
 786                  * page_length = bytes to copy for this page
 787                  */
 788                 gtt_page_base = offset & PAGE_MASK;
 789                 gtt_page_offset = offset & ~PAGE_MASK;
 790                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 791                 data_page_offset = data_ptr & ~PAGE_MASK;
 792
 793                 page_length = remain;
 794                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 795                         page_length = PAGE_SIZE - gtt_page_offset;
 796                 if ((data_page_offset + page_length) > PAGE_SIZE)
 797                         page_length = PAGE_SIZE - data_page_offset;
 798
 799                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 800                                   gtt_page_base, gtt_page_offset,
 801                                   user_pages[data_page_index],
 802                                   data_page_offset,
 803                                   page_length);
 804
 805                 remain -= page_length;
 806                 offset += page_length;
 807                 data_ptr += page_length;
 808         }
 809
 810 out_unpin_pages:
 811         for (i = 0; i < pinned_pages; i++)
 812                 page_cache_release(user_pages[i]);
 813         drm_free_large(user_pages);
 814
 815         return ret;
 816 }
 817
 818 /**
 819  * This is the fast shmem pwrite path, which attempts to directly
 820  * copy_from_user into the kmapped pages backing the object.
 821  */
 822 static int
 823 i915_gem_shmem_pwrite_fast(struct drm_device *dev,
 824                            struct drm_i915_gem_object *obj,
 825                            struct drm_i915_gem_pwrite *args,
 826                            struct drm_file *file)
 827 {
 828         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 829         ssize_t remain;
 830         loff_t offset;
 831         char __user *user_data;
 832         int page_offset, page_length;
 833
 834         user_data = (char __user *) (uintptr_t) args->data_ptr;
 835         remain = args->size;
 836
 837         offset = args->offset;
 838         obj->dirty = 1;
 839
 840         while (remain > 0) {
 841                 struct page *page;
 842                 char *vaddr;
 843                 int ret;
 844
 845                 /* Operation in this page
 846                  *
 847                  * page_offset = offset within page
 848                  * page_length = bytes to copy for this page
 849                  */
 850                 page_offset = offset & (PAGE_SIZE-1);
 851                 page_length = remain;
 852                 if ((page_offset + remain) > PAGE_SIZE)
 853                         page_length = PAGE_SIZE - page_offset;
 854
 855                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 856                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 857                 if (IS_ERR(page))
 858                         return PTR_ERR(page);
 859
 860                 vaddr = kmap_atomic(page, KM_USER0);
 861                 ret = __copy_from_user_inatomic(vaddr + page_offset,
 862                                                 user_data,
 863                                                 page_length);
 864                 kunmap_atomic(vaddr, KM_USER0);
 865
 866                 set_page_dirty(page);
 867                 mark_page_accessed(page);
 868                 page_cache_release(page);
 869
 870                 /* If we get a fault while copying data, then (presumably) our
 871                  * source page isn't available.  Return the error and we'll
 872                  * retry in the slow path.
 873                  */
 874                 if (ret)
 875                         return -EFAULT;
 876
 877                 remain -= page_length;
 878                 user_data += page_length;
 879                 offset += page_length;
 880         }
 881
 882         return 0;
 883 }
 884
 885 /**
 886  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 887  * the memory and maps it using kmap_atomic for copying.
 888  *
 889  * This avoids taking mmap_sem for faulting on the user's address while the
 890  * struct_mutex is held.
 891  */
 892 static int
 893 i915_gem_shmem_pwrite_slow(struct drm_device *dev,
 894                            struct drm_i915_gem_object *obj,
 895                            struct drm_i915_gem_pwrite *args,
 896                            struct drm_file *file)
 897 {
 898         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
 899         struct mm_struct *mm = current->mm;
 900         struct page **user_pages;
 901         ssize_t remain;
 902         loff_t offset, pinned_pages, i;
 903         loff_t first_data_page, last_data_page, num_pages;
 904         int shmem_page_offset;
 905         int data_page_index,  data_page_offset;
 906         int page_length;
 907         int ret;
 908         uint64_t data_ptr = args->data_ptr;
 909         int do_bit17_swizzling;
 910
 911         remain = args->size;
 912
 913         /* Pin the user pages containing the data.  We can't fault while
 914          * holding the struct mutex, and all of the pwrite implementations
 915          * want to hold it while dereferencing the user data.
 916          */
 917         first_data_page = data_ptr / PAGE_SIZE;
 918         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 919         num_pages = last_data_page - first_data_page + 1;
 920
 921         user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
 922         if (user_pages == NULL)
 923                 return -ENOMEM;
 924
 925         mutex_unlock(&dev->struct_mutex);
 926         down_read(&mm->mmap_sem);
 927         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 928                                       num_pages, 0, 0, user_pages, NULL);
 929         up_read(&mm->mmap_sem);
 930         mutex_lock(&dev->struct_mutex);
 931         if (pinned_pages < num_pages) {
 932                 ret = -EFAULT;
 933                 goto out;
 934         }
 935
 936         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 937         if (ret)
 938                 goto out;
 939
 940         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 941
 942         offset = args->offset;
 943         obj->dirty = 1;
 944
 945         while (remain > 0) {
 946                 struct page *page;
 947
 948                 /* Operation in this page
 949                  *
 950                  * shmem_page_offset = offset within page in shmem file
 951                  * data_page_index = page number in get_user_pages return
 952                  * data_page_offset = offset with data_page_index page.
 953                  * page_length = bytes to copy for this page
 954                  */
 955                 shmem_page_offset = offset & ~PAGE_MASK;
 956                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 957                 data_page_offset = data_ptr & ~PAGE_MASK;
 958
 959                 page_length = remain;
 960                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 961                         page_length = PAGE_SIZE - shmem_page_offset;
 962                 if ((data_page_offset + page_length) > PAGE_SIZE)
 963                         page_length = PAGE_SIZE - data_page_offset;
 964
 965                 page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT,
 966                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
 967                 if (IS_ERR(page)) {
 968                         ret = PTR_ERR(page);
 969                         goto out;
 970                 }
 971
 972                 if (do_bit17_swizzling) {
 973                         slow_shmem_bit17_copy(page,
 974                                               shmem_page_offset,
 975                                               user_pages[data_page_index],
 976                                               data_page_offset,
 977                                               page_length,
 978                                               0);
 979                 } else {
 980                         slow_shmem_copy(page,
 981                                         shmem_page_offset,
 982                                         user_pages[data_page_index],
 983                                         data_page_offset,
 984                                         page_length);
 985                 }
 986
 987                 set_page_dirty(page);
 988                 mark_page_accessed(page);
 989                 page_cache_release(page);
 990
 991                 remain -= page_length;
 992                 data_ptr += page_length;
 993                 offset += page_length;
 994         }
 995
 996 out:
 997         for (i = 0; i < pinned_pages; i++)
 998                 page_cache_release(user_pages[i]);
 999         drm_free_large(user_pages);
1000
1001         return ret;
1002 }
1003
1004 /**
1005  * Writes data to the object referenced by handle.
1006  *
1007  * On error, the contents of the buffer that were to be modified are undefined.
1008  */
1009 int
1010 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1011                       struct drm_file *file)
1012 {
1013         struct drm_i915_gem_pwrite *args = data;
1014         struct drm_i915_gem_object *obj;
1015         int ret;
1016
1017         if (args->size == 0)
1018                 return 0;
1019
1020         if (!access_ok(VERIFY_READ,
1021                        (char __user *)(uintptr_t)args->data_ptr,
1022                        args->size))
1023                 return -EFAULT;
1024
1025         ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
1026                                       args->size);
1027         if (ret)
1028                 return -EFAULT;
1029
1030         ret = i915_mutex_lock_interruptible(dev);
1031         if (ret)
1032                 return ret;
1033
1034         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1035         if (obj == NULL) {
1036                 ret = -ENOENT;
1037                 goto unlock;
1038         }
1039
1040         /* Bounds check destination. */
1041         if (args->offset > obj->base.size ||
1042             args->size > obj->base.size - args->offset) {
1043                 ret = -EINVAL;
1044                 goto out;
1045         }
1046
1047         /* We can only do the GTT pwrite on untiled buffers, as otherwise
1048          * it would end up going through the fenced access, and we'll get
1049          * different detiling behavior between reading and writing.
1050          * pread/pwrite currently are reading and writing from the CPU
1051          * perspective, requiring manual detiling by the client.
1052          */
1053         if (obj->phys_obj)
1054                 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1055         else if (obj->tiling_mode == I915_TILING_NONE &&
1056                  obj->gtt_space &&
1057                  obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1058                 ret = i915_gem_object_pin(obj, 0, true);
1059                 if (ret)
1060                         goto out;
1061
1062                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1063                 if (ret)
1064                         goto out_unpin;
1065
1066                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1067                 if (ret == -EFAULT)
1068                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1069
1070 out_unpin:
1071                 i915_gem_object_unpin(obj);
1072         } else {
1073                 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1074                 if (ret)
1075                         goto out;
1076
1077                 ret = -EFAULT;
1078                 if (!i915_gem_object_needs_bit17_swizzle(obj))
1079                         ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1080                 if (ret == -EFAULT)
1081                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1082         }
1083
1084 out:
1085         drm_gem_object_unreference(&obj->base);
1086 unlock:
1087         mutex_unlock(&dev->struct_mutex);
1088         return ret;
1089 }
1090
1091 /**
1092  * Called when user space prepares to use an object with the CPU, either
1093  * through the mmap ioctl's mapping or a GTT mapping.
1094  */
1095 int
1096 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1097                           struct drm_file *file)
1098 {
1099         struct drm_i915_private *dev_priv = dev->dev_private;
1100         struct drm_i915_gem_set_domain *args = data;
1101         struct drm_i915_gem_object *obj;
1102         uint32_t read_domains = args->read_domains;
1103         uint32_t write_domain = args->write_domain;
1104         int ret;
1105
1106         if (!(dev->driver->driver_features & DRIVER_GEM))
1107                 return -ENODEV;
1108
1109         /* Only handle setting domains to types used by the CPU. */
1110         if (write_domain & I915_GEM_GPU_DOMAINS)
1111                 return -EINVAL;
1112
1113         if (read_domains & I915_GEM_GPU_DOMAINS)
1114                 return -EINVAL;
1115
1116         /* Having something in the write domain implies it's in the read
1117          * domain, and only that read domain.  Enforce that in the request.
1118          */
1119         if (write_domain != 0 && read_domains != write_domain)
1120                 return -EINVAL;
1121
1122         ret = i915_mutex_lock_interruptible(dev);
1123         if (ret)
1124                 return ret;
1125
1126         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1127         if (obj == NULL) {
1128                 ret = -ENOENT;
1129                 goto unlock;
1130         }
1131
1132         intel_mark_busy(dev, obj);
1133
1134         if (read_domains & I915_GEM_DOMAIN_GTT) {
1135                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1136
1137                 /* Update the LRU on the fence for the CPU access that's
1138                  * about to occur.
1139                  */
1140                 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1141                         struct drm_i915_fence_reg *reg =
1142                                 &dev_priv->fence_regs[obj->fence_reg];
1143                         list_move_tail(&reg->lru_list,
1144                                        &dev_priv->mm.fence_list);
1145                 }
1146
1147                 /* Silently promote "you're not bound, there was nothing to do"
1148                  * to success, since the client was just asking us to
1149                  * make sure everything was done.
1150                  */
1151                 if (ret == -EINVAL)
1152                         ret = 0;
1153         } else {
1154                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1155         }
1156
1157         /* Maintain LRU order of "inactive" objects */
1158         if (ret == 0 && i915_gem_object_is_inactive(obj))
1159                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1160
1161         drm_gem_object_unreference(&obj->base);
1162 unlock:
1163         mutex_unlock(&dev->struct_mutex);
1164         return ret;
1165 }
1166
1167 /**
1168  * Called when user space has done writes to this buffer
1169  */
1170 int
1171 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1172                          struct drm_file *file)
1173 {
1174         struct drm_i915_gem_sw_finish *args = data;
1175         struct drm_i915_gem_object *obj;
1176         int ret = 0;
1177
1178         if (!(dev->driver->driver_features & DRIVER_GEM))
1179                 return -ENODEV;
1180
1181         ret = i915_mutex_lock_interruptible(dev);
1182         if (ret)
1183                 return ret;
1184
1185         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1186         if (obj == NULL) {
1187                 ret = -ENOENT;
1188                 goto unlock;
1189         }
1190
1191         /* Pinned buffers may be scanout, so flush the cache */
1192         if (obj->pin_count)
1193                 i915_gem_object_flush_cpu_write_domain(obj);
1194
1195         drm_gem_object_unreference(&obj->base);
1196 unlock:
1197         mutex_unlock(&dev->struct_mutex);
1198         return ret;
1199 }
1200
1201 /**
1202  * Maps the contents of an object, returning the address it is mapped
1203  * into.
1204  *
1205  * While the mapping holds a reference on the contents of the object, it doesn't
1206  * imply a ref on the object itself.
1207  */
1208 int
1209 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1210                     struct drm_file *file)
1211 {
1212         struct drm_i915_private *dev_priv = dev->dev_private;
1213         struct drm_i915_gem_mmap *args = data;
1214         struct drm_gem_object *obj;
1215         loff_t offset;
1216         unsigned long addr;
1217
1218         if (!(dev->driver->driver_features & DRIVER_GEM))
1219                 return -ENODEV;
1220
1221         obj = drm_gem_object_lookup(dev, file, args->handle);
1222         if (obj == NULL)
1223                 return -ENOENT;
1224
1225         if (obj->size > dev_priv->mm.gtt_mappable_end) {
1226                 drm_gem_object_unreference_unlocked(obj);
1227                 return -E2BIG;
1228         }
1229
1230         offset = args->offset;
1231
1232         down_write(&current->mm->mmap_sem);
1233         addr = do_mmap(obj->filp, 0, args->size,
1234                        PROT_READ | PROT_WRITE, MAP_SHARED,
1235                        args->offset);
1236         up_write(&current->mm->mmap_sem);
1237         drm_gem_object_unreference_unlocked(obj);
1238         if (IS_ERR((void *)addr))
1239                 return addr;
1240
1241         args->addr_ptr = (uint64_t) addr;
1242
1243         return 0;
1244 }
1245
1246 /**
1247  * i915_gem_fault - fault a page into the GTT
1248  * vma: VMA in question
1249  * vmf: fault info
1250  *
1251  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1252  * from userspace.  The fault handler takes care of binding the object to
1253  * the GTT (if needed), allocating and programming a fence register (again,
1254  * only if needed based on whether the old reg is still valid or the object
1255  * is tiled) and inserting a new PTE into the faulting process.
1256  *
1257  * Note that the faulting process may involve evicting existing objects
1258  * from the GTT and/or fence registers to make room.  So performance may
1259  * suffer if the GTT working set is large or there are few fence registers
1260  * left.
1261  */
1262 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1263 {
1264         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1265         struct drm_device *dev = obj->base.dev;
1266         drm_i915_private_t *dev_priv = dev->dev_private;
1267         pgoff_t page_offset;
1268         unsigned long pfn;
1269         int ret = 0;
1270         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1271
1272         /* We don't use vmf->pgoff since that has the fake offset */
1273         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1274                 PAGE_SHIFT;
1275
1276         /* Now bind it into the GTT if needed */
1277         mutex_lock(&dev->struct_mutex);
1278         BUG_ON(obj->pin_count && !obj->pin_mappable);
1279
1280         if (obj->gtt_space) {
1281                 if (!obj->map_and_fenceable) {
1282                         ret = i915_gem_object_unbind(obj);
1283                         if (ret)
1284                                 goto unlock;
1285                 }
1286         }
1287
1288         if (!obj->gtt_space) {
1289                 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1290                 if (ret)
1291                         goto unlock;
1292         }
1293
1294         ret = i915_gem_object_set_to_gtt_domain(obj, write);
1295         if (ret)
1296                 goto unlock;
1297
1298         if (!obj->fault_mappable) {
1299                 obj->fault_mappable = true;
1300                 i915_gem_info_update_mappable(dev_priv, obj, true);
1301         }
1302
1303         /* Need a new fence register? */
1304         if (obj->tiling_mode != I915_TILING_NONE) {
1305                 ret = i915_gem_object_get_fence_reg(obj, true);
1306                 if (ret)
1307                         goto unlock;
1308         }
1309
1310         if (i915_gem_object_is_inactive(obj))
1311                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1312
1313         pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1314                 page_offset;
1315
1316         /* Finally, remap it using the new GTT offset */
1317         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1318 unlock:
1319         mutex_unlock(&dev->struct_mutex);
1320
1321         switch (ret) {
1322         case -EAGAIN:
1323                 set_need_resched();
1324         case 0:
1325         case -ERESTARTSYS:
1326                 return VM_FAULT_NOPAGE;
1327         case -ENOMEM:
1328                 return VM_FAULT_OOM;
1329         default:
1330                 return VM_FAULT_SIGBUS;
1331         }
1332 }
1333
1334 /**
1335  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1336  * @obj: obj in question
1337  *
1338  * GEM memory mapping works by handing back to userspace a fake mmap offset
1339  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1340  * up the object based on the offset and sets up the various memory mapping
1341  * structures.
1342  *
1343  * This routine allocates and attaches a fake offset for @obj.
1344  */
1345 static int
1346 i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
1347 {
1348         struct drm_device *dev = obj->base.dev;
1349         struct drm_gem_mm *mm = dev->mm_private;
1350         struct drm_map_list *list;
1351         struct drm_local_map *map;
1352         int ret = 0;
1353
1354         /* Set the object up for mmap'ing */
1355         list = &obj->base.map_list;
1356         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1357         if (!list->map)
1358                 return -ENOMEM;
1359
1360         map = list->map;
1361         map->type = _DRM_GEM;
1362         map->size = obj->base.size;
1363         map->handle = obj;
1364
1365         /* Get a DRM GEM mmap offset allocated... */
1366         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1367                                                     obj->base.size / PAGE_SIZE,
1368                                                     0, 0);
1369         if (!list->file_offset_node) {
1370                 DRM_ERROR("failed to allocate offset for bo %d\n",
1371                           obj->base.name);
1372                 ret = -ENOSPC;
1373                 goto out_free_list;
1374         }
1375
1376         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1377                                                   obj->base.size / PAGE_SIZE,
1378                                                   0);
1379         if (!list->file_offset_node) {
1380                 ret = -ENOMEM;
1381                 goto out_free_list;
1382         }
1383
1384         list->hash.key = list->file_offset_node->start;
1385         ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1386         if (ret) {
1387                 DRM_ERROR("failed to add to map hash\n");
1388                 goto out_free_mm;
1389         }
1390
1391         return 0;
1392
1393 out_free_mm:
1394         drm_mm_put_block(list->file_offset_node);
1395 out_free_list:
1396         kfree(list->map);
1397         list->map = NULL;
1398
1399         return ret;
1400 }
1401
1402 /**
1403  * i915_gem_release_mmap - remove physical page mappings
1404  * @obj: obj in question
1405  *
1406  * Preserve the reservation of the mmapping with the DRM core code, but
1407  * relinquish ownership of the pages back to the system.
1408  *
1409  * It is vital that we remove the page mapping if we have mapped a tiled
1410  * object through the GTT and then lose the fence register due to
1411  * resource pressure. Similarly if the object has been moved out of the
1412  * aperture, than pages mapped into userspace must be revoked. Removing the
1413  * mapping will then trigger a page fault on the next user access, allowing
1414  * fixup by i915_gem_fault().
1415  */
1416 void
1417 i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1418 {
1419         struct drm_device *dev = obj->base.dev;
1420         struct drm_i915_private *dev_priv = dev->dev_private;
1421
1422         if (unlikely(obj->base.map_list.map && dev->dev_mapping))
1423                 unmap_mapping_range(dev->dev_mapping,
1424                                     (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1425                                     obj->base.size, 1);
1426
1427         if (obj->fault_mappable) {
1428                 obj->fault_mappable = false;
1429                 i915_gem_info_update_mappable(dev_priv, obj, false);
1430         }
1431 }
1432
1433 static void
1434 i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
1435 {
1436         struct drm_device *dev = obj->base.dev;
1437         struct drm_gem_mm *mm = dev->mm_private;
1438         struct drm_map_list *list = &obj->base.map_list;
1439
1440         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1441         drm_mm_put_block(list->file_offset_node);
1442         kfree(list->map);
1443         list->map = NULL;
1444 }
1445
1446 /**
1447  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1448  * @obj: object to check
1449  *
1450  * Return the required GTT alignment for an object, taking into account
1451  * potential fence register mapping.
1452  */
1453 static uint32_t
1454 i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj)
1455 {
1456         struct drm_device *dev = obj->base.dev;
1457
1458         /*
1459          * Minimum alignment is 4k (GTT page size), but might be greater
1460          * if a fence register is needed for the object.
1461          */
1462         if (INTEL_INFO(dev)->gen >= 4 ||
1463             obj->tiling_mode == I915_TILING_NONE)
1464                 return 4096;
1465
1466         /*
1467          * Previous chips need to be aligned to the size of the smallest
1468          * fence register that can contain the object.
1469          */
1470         return i915_gem_get_gtt_size(obj);
1471 }
1472
1473 /**
1474  * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1475  *                                       unfenced object
1476  * @obj: object to check
1477  *
1478  * Return the required GTT alignment for an object, only taking into account
1479  * unfenced tiled surface requirements.
1480  */
1481 static uint32_t
1482 i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj)
1483 {
1484         struct drm_device *dev = obj->base.dev;
1485         int tile_height;
1486
1487         /*
1488          * Minimum alignment is 4k (GTT page size) for sane hw.
1489          */
1490         if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1491             obj->tiling_mode == I915_TILING_NONE)
1492                 return 4096;
1493
1494         /*
1495          * Older chips need unfenced tiled buffers to be aligned to the left
1496          * edge of an even tile row (where tile rows are counted as if the bo is
1497          * placed in a fenced gtt region).
1498          */
1499         if (IS_GEN2(dev) ||
1500             (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)))
1501                 tile_height = 32;
1502         else
1503                 tile_height = 8;
1504
1505         return tile_height * obj->stride * 2;
1506 }
1507
1508 static uint32_t
1509 i915_gem_get_gtt_size(struct drm_i915_gem_object *obj)
1510 {
1511         struct drm_device *dev = obj->base.dev;
1512         uint32_t size;
1513
1514         /*
1515          * Minimum alignment is 4k (GTT page size), but might be greater
1516          * if a fence register is needed for the object.
1517          */
1518         if (INTEL_INFO(dev)->gen >= 4)
1519                 return obj->base.size;
1520
1521         /*
1522          * Previous chips need to be aligned to the size of the smallest
1523          * fence register that can contain the object.
1524          */
1525         if (INTEL_INFO(dev)->gen == 3)
1526                 size = 1024*1024;
1527         else
1528                 size = 512*1024;
1529
1530         while (size < obj->base.size)
1531                 size <<= 1;
1532
1533         return size;
1534 }
1535
1536 /**
1537  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1538  * @dev: DRM device
1539  * @data: GTT mapping ioctl data
1540  * @file: GEM object info
1541  *
1542  * Simply returns the fake offset to userspace so it can mmap it.
1543  * The mmap call will end up in drm_gem_mmap(), which will set things
1544  * up so we can get faults in the handler above.
1545  *
1546  * The fault handler will take care of binding the object into the GTT
1547  * (since it may have been evicted to make room for something), allocating
1548  * a fence register, and mapping the appropriate aperture address into
1549  * userspace.
1550  */
1551 int
1552 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1553                         struct drm_file *file)
1554 {
1555         struct drm_i915_private *dev_priv = dev->dev_private;
1556         struct drm_i915_gem_mmap_gtt *args = data;
1557         struct drm_i915_gem_object *obj;
1558         int ret;
1559
1560         if (!(dev->driver->driver_features & DRIVER_GEM))
1561                 return -ENODEV;
1562
1563         ret = i915_mutex_lock_interruptible(dev);
1564         if (ret)
1565                 return ret;
1566
1567         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1568         if (obj == NULL) {
1569                 ret = -ENOENT;
1570                 goto unlock;
1571         }
1572
1573         if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1574                 ret = -E2BIG;
1575                 goto unlock;
1576         }
1577
1578         if (obj->madv != I915_MADV_WILLNEED) {
1579                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1580                 ret = -EINVAL;
1581                 goto out;
1582         }
1583
1584         if (!obj->base.map_list.map) {
1585                 ret = i915_gem_create_mmap_offset(obj);
1586                 if (ret)
1587                         goto out;
1588         }
1589
1590         args->offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1591
1592 out:
1593         drm_gem_object_unreference(&obj->base);
1594 unlock:
1595         mutex_unlock(&dev->struct_mutex);
1596         return ret;
1597 }
1598
1599 static int
1600 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1601                               gfp_t gfpmask)
1602 {
1603         int page_count, i;
1604         struct address_space *mapping;
1605         struct inode *inode;
1606         struct page *page;
1607
1608         /* Get the list of pages out of our struct file.  They'll be pinned
1609          * at this point until we release them.
1610          */
1611         page_count = obj->base.size / PAGE_SIZE;
1612         BUG_ON(obj->pages != NULL);
1613         obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1614         if (obj->pages == NULL)
1615                 return -ENOMEM;
1616
1617         inode = obj->base.filp->f_path.dentry->d_inode;
1618         mapping = inode->i_mapping;
1619         for (i = 0; i < page_count; i++) {
1620                 page = read_cache_page_gfp(mapping, i,
1621                                            GFP_HIGHUSER |
1622                                            __GFP_COLD |
1623                                            __GFP_RECLAIMABLE |
1624                                            gfpmask);
1625                 if (IS_ERR(page))
1626                         goto err_pages;
1627
1628                 obj->pages[i] = page;
1629         }
1630
1631         if (obj->tiling_mode != I915_TILING_NONE)
1632                 i915_gem_object_do_bit_17_swizzle(obj);
1633
1634         return 0;
1635
1636 err_pages:
1637         while (i--)
1638                 page_cache_release(obj->pages[i]);
1639
1640         drm_free_large(obj->pages);
1641         obj->pages = NULL;
1642         return PTR_ERR(page);
1643 }
1644
1645 static void
1646 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1647 {
1648         int page_count = obj->base.size / PAGE_SIZE;
1649         int i;
1650
1651         BUG_ON(obj->madv == __I915_MADV_PURGED);
1652
1653         if (obj->tiling_mode != I915_TILING_NONE)
1654                 i915_gem_object_save_bit_17_swizzle(obj);
1655
1656         if (obj->madv == I915_MADV_DONTNEED)
1657                 obj->dirty = 0;
1658
1659         for (i = 0; i < page_count; i++) {
1660                 if (obj->dirty)
1661                         set_page_dirty(obj->pages[i]);
1662
1663                 if (obj->madv == I915_MADV_WILLNEED)
1664                         mark_page_accessed(obj->pages[i]);
1665
1666                 page_cache_release(obj->pages[i]);
1667         }
1668         obj->dirty = 0;
1669
1670         drm_free_large(obj->pages);
1671         obj->pages = NULL;
1672 }
1673
1674 static uint32_t
1675 i915_gem_next_request_seqno(struct drm_device *dev,
1676                             struct intel_ring_buffer *ring)
1677 {
1678         drm_i915_private_t *dev_priv = dev->dev_private;
1679         return ring->outstanding_lazy_request = dev_priv->next_seqno;
1680 }
1681
1682 static void
1683 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1684                                struct intel_ring_buffer *ring)
1685 {
1686         struct drm_device *dev = obj->base.dev;
1687         struct drm_i915_private *dev_priv = dev->dev_private;
1688         uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1689
1690         BUG_ON(ring == NULL);
1691         obj->ring = ring;
1692
1693         /* Add a reference if we're newly entering the active list. */
1694         if (!obj->active) {
1695                 drm_gem_object_reference(&obj->base);
1696                 obj->active = 1;
1697         }
1698
1699         /* Move from whatever list we were on to the tail of execution. */
1700         list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1701         list_move_tail(&obj->ring_list, &ring->active_list);
1702         obj->last_rendering_seqno = seqno;
1703 }
1704
1705 static void
1706 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1707 {
1708         struct drm_device *dev = obj->base.dev;
1709         drm_i915_private_t *dev_priv = dev->dev_private;
1710
1711         BUG_ON(!obj->active);
1712         list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1713         list_del_init(&obj->ring_list);
1714         obj->last_rendering_seqno = 0;
1715 }
1716
1717 /* Immediately discard the backing storage */
1718 static void
1719 i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1720 {
1721         struct inode *inode;
1722
1723         /* Our goal here is to return as much of the memory as
1724          * is possible back to the system as we are called from OOM.
1725          * To do this we must instruct the shmfs to drop all of its
1726          * backing pages, *now*. Here we mirror the actions taken
1727          * when by shmem_delete_inode() to release the backing store.
1728          */
1729         inode = obj->base.filp->f_path.dentry->d_inode;
1730         truncate_inode_pages(inode->i_mapping, 0);
1731         if (inode->i_op->truncate_range)
1732                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1733
1734         obj->madv = __I915_MADV_PURGED;
1735 }
1736
1737 static inline int
1738 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1739 {
1740         return obj->madv == I915_MADV_DONTNEED;
1741 }
1742
1743 static void
1744 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1745 {
1746         struct drm_device *dev = obj->base.dev;
1747         drm_i915_private_t *dev_priv = dev->dev_private;
1748
1749         if (obj->pin_count != 0)
1750                 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1751         else
1752                 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1753         list_del_init(&obj->ring_list);
1754
1755         BUG_ON(!list_empty(&obj->gpu_write_list));
1756
1757         obj->last_rendering_seqno = 0;
1758         obj->ring = NULL;
1759         if (obj->active) {
1760                 obj->active = 0;
1761                 drm_gem_object_unreference(&obj->base);
1762         }
1763         WARN_ON(i915_verify_lists(dev));
1764 }
1765
1766 static void
1767 i915_gem_process_flushing_list(struct drm_device *dev,
1768                                uint32_t flush_domains,
1769                                struct intel_ring_buffer *ring)
1770 {
1771         drm_i915_private_t *dev_priv = dev->dev_private;
1772         struct drm_i915_gem_object *obj, *next;
1773
1774         list_for_each_entry_safe(obj, next,
1775                                  &ring->gpu_write_list,
1776                                  gpu_write_list) {
1777                 if (obj->base.write_domain & flush_domains) {
1778                         uint32_t old_write_domain = obj->base.write_domain;
1779
1780                         obj->base.write_domain = 0;
1781                         list_del_init(&obj->gpu_write_list);
1782                         i915_gem_object_move_to_active(obj, ring);
1783
1784                         /* update the fence lru list */
1785                         if (obj->fence_reg != I915_FENCE_REG_NONE) {
1786                                 struct drm_i915_fence_reg *reg =
1787                                         &dev_priv->fence_regs[obj->fence_reg];
1788                                 list_move_tail(&reg->lru_list,
1789                                                 &dev_priv->mm.fence_list);
1790                         }
1791
1792                         trace_i915_gem_object_change_domain(obj,
1793                                                             obj->base.read_domains,
1794                                                             old_write_domain);
1795                 }
1796         }
1797 }
1798
1799 int
1800 i915_add_request(struct drm_device *dev,
1801                  struct drm_file *file,
1802                  struct drm_i915_gem_request *request,
1803                  struct intel_ring_buffer *ring)
1804 {
1805         drm_i915_private_t *dev_priv = dev->dev_private;
1806         struct drm_i915_file_private *file_priv = NULL;
1807         uint32_t seqno;
1808         int was_empty;
1809         int ret;
1810
1811         BUG_ON(request == NULL);
1812
1813         if (file != NULL)
1814                 file_priv = file->driver_priv;
1815
1816         ret = ring->add_request(ring, &seqno);
1817         if (ret)
1818             return ret;
1819
1820         ring->outstanding_lazy_request = false;
1821
1822         request->seqno = seqno;
1823         request->ring = ring;
1824         request->emitted_jiffies = jiffies;
1825         was_empty = list_empty(&ring->request_list);
1826         list_add_tail(&request->list, &ring->request_list);
1827
1828         if (file_priv) {
1829                 spin_lock(&file_priv->mm.lock);
1830                 request->file_priv = file_priv;
1831                 list_add_tail(&request->client_list,
1832                               &file_priv->mm.request_list);
1833                 spin_unlock(&file_priv->mm.lock);
1834         }
1835
1836         if (!dev_priv->mm.suspended) {
1837                 mod_timer(&dev_priv->hangcheck_timer,
1838                           jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1839                 if (was_empty)
1840                         queue_delayed_work(dev_priv->wq,
1841                                            &dev_priv->mm.retire_work, HZ);
1842         }
1843         return 0;
1844 }
1845
1846 /**
1847  * Command execution barrier
1848  *
1849  * Ensures that all commands in the ring are finished
1850  * before signalling the CPU
1851  */
1852 static void
1853 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1854 {
1855         uint32_t flush_domains = 0;
1856
1857         /* The sampler always gets flushed on i965 (sigh) */
1858         if (INTEL_INFO(dev)->gen >= 4)
1859                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1860
1861         ring->flush(ring, I915_GEM_DOMAIN_COMMAND, flush_domains);
1862 }
1863
1864 static inline void
1865 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1866 {
1867         struct drm_i915_file_private *file_priv = request->file_priv;
1868
1869         if (!file_priv)
1870                 return;
1871
1872         spin_lock(&file_priv->mm.lock);
1873         list_del(&request->client_list);
1874         request->file_priv = NULL;
1875         spin_unlock(&file_priv->mm.lock);
1876 }
1877
1878 static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1879                                       struct intel_ring_buffer *ring)
1880 {
1881         while (!list_empty(&ring->request_list)) {
1882                 struct drm_i915_gem_request *request;
1883
1884                 request = list_first_entry(&ring->request_list,
1885                                            struct drm_i915_gem_request,
1886                                            list);
1887
1888                 list_del(&request->list);
1889                 i915_gem_request_remove_from_client(request);
1890                 kfree(request);
1891         }
1892
1893         while (!list_empty(&ring->active_list)) {
1894                 struct drm_i915_gem_object *obj;
1895
1896                 obj = list_first_entry(&ring->active_list,
1897                                        struct drm_i915_gem_object,
1898                                        ring_list);
1899
1900                 obj->base.write_domain = 0;
1901                 list_del_init(&obj->gpu_write_list);
1902                 i915_gem_object_move_to_inactive(obj);
1903         }
1904 }
1905
1906 void i915_gem_reset(struct drm_device *dev)
1907 {
1908         struct drm_i915_private *dev_priv = dev->dev_private;
1909         struct drm_i915_gem_object *obj;
1910         int i;
1911
1912         i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1913         i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1914         i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring);
1915
1916         /* Remove anything from the flushing lists. The GPU cache is likely
1917          * to be lost on reset along with the data, so simply move the
1918          * lost bo to the inactive list.
1919          */
1920         while (!list_empty(&dev_priv->mm.flushing_list)) {
1921                 obj= list_first_entry(&dev_priv->mm.flushing_list,
1922                                       struct drm_i915_gem_object,
1923                                       mm_list);
1924
1925                 obj->base.write_domain = 0;
1926                 list_del_init(&obj->gpu_write_list);
1927                 i915_gem_object_move_to_inactive(obj);
1928         }
1929
1930         /* Move everything out of the GPU domains to ensure we do any
1931          * necessary invalidation upon reuse.
1932          */
1933         list_for_each_entry(obj,
1934                             &dev_priv->mm.inactive_list,
1935                             mm_list)
1936         {
1937                 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1938         }
1939
1940         /* The fence registers are invalidated so clear them out */
1941         for (i = 0; i < 16; i++) {
1942                 struct drm_i915_fence_reg *reg;
1943
1944                 reg = &dev_priv->fence_regs[i];
1945                 if (!reg->obj)
1946                         continue;
1947
1948                 i915_gem_clear_fence_reg(reg->obj);
1949         }
1950 }
1951
1952 /**
1953  * This function clears the request list as sequence numbers are passed.
1954  */
1955 static void
1956 i915_gem_retire_requests_ring(struct drm_device *dev,
1957                               struct intel_ring_buffer *ring)
1958 {
1959         drm_i915_private_t *dev_priv = dev->dev_private;
1960         uint32_t seqno;
1961
1962         if (!ring->status_page.page_addr ||
1963             list_empty(&ring->request_list))
1964                 return;
1965
1966         WARN_ON(i915_verify_lists(dev));
1967
1968         seqno = ring->get_seqno(ring);
1969         while (!list_empty(&ring->request_list)) {
1970                 struct drm_i915_gem_request *request;
1971
1972                 request = list_first_entry(&ring->request_list,
1973                                            struct drm_i915_gem_request,
1974                                            list);
1975
1976                 if (!i915_seqno_passed(seqno, request->seqno))
1977                         break;
1978
1979                 trace_i915_gem_request_retire(dev, request->seqno);
1980
1981                 list_del(&request->list);
1982                 i915_gem_request_remove_from_client(request);
1983                 kfree(request);
1984         }
1985
1986         /* Move any buffers on the active list that are no longer referenced
1987          * by the ringbuffer to the flushing/inactive lists as appropriate.
1988          */
1989         while (!list_empty(&ring->active_list)) {
1990                 struct drm_i915_gem_object *obj;
1991
1992                 obj= list_first_entry(&ring->active_list,
1993                                       struct drm_i915_gem_object,
1994                                       ring_list);
1995
1996                 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1997                         break;
1998
1999                 if (obj->base.write_domain != 0)
2000                         i915_gem_object_move_to_flushing(obj);
2001                 else
2002                         i915_gem_object_move_to_inactive(obj);
2003         }
2004
2005         if (unlikely (dev_priv->trace_irq_seqno &&
2006                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
2007                 ring->user_irq_put(ring);
2008                 dev_priv->trace_irq_seqno = 0;
2009         }
2010
2011         WARN_ON(i915_verify_lists(dev));
2012 }
2013
2014 void
2015 i915_gem_retire_requests(struct drm_device *dev)
2016 {
2017         drm_i915_private_t *dev_priv = dev->dev_private;
2018
2019         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2020             struct drm_i915_gem_object *obj, *next;
2021
2022             /* We must be careful that during unbind() we do not
2023              * accidentally infinitely recurse into retire requests.
2024              * Currently:
2025              *   retire -> free -> unbind -> wait -> retire_ring
2026              */
2027             list_for_each_entry_safe(obj, next,
2028                                      &dev_priv->mm.deferred_free_list,
2029                                      mm_list)
2030                     i915_gem_free_object_tail(obj);
2031         }
2032
2033         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
2034         i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
2035         i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring);
2036 }
2037
2038 static void
2039 i915_gem_retire_work_handler(struct work_struct *work)
2040 {
2041         drm_i915_private_t *dev_priv;
2042         struct drm_device *dev;
2043
2044         dev_priv = container_of(work, drm_i915_private_t,
2045                                 mm.retire_work.work);
2046         dev = dev_priv->dev;
2047
2048         /* Come back later if the device is busy... */
2049         if (!mutex_trylock(&dev->struct_mutex)) {
2050                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2051                 return;
2052         }
2053
2054         i915_gem_retire_requests(dev);
2055
2056         if (!dev_priv->mm.suspended &&
2057                 (!list_empty(&dev_priv->render_ring.request_list) ||
2058                  !list_empty(&dev_priv->bsd_ring.request_list) ||
2059                  !list_empty(&dev_priv->blt_ring.request_list)))
2060                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2061         mutex_unlock(&dev->struct_mutex);
2062 }
2063
2064 int
2065 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
2066                      bool interruptible, struct intel_ring_buffer *ring)
2067 {
2068         drm_i915_private_t *dev_priv = dev->dev_private;
2069         u32 ier;
2070         int ret = 0;
2071
2072         BUG_ON(seqno == 0);
2073
2074         if (atomic_read(&dev_priv->mm.wedged))
2075                 return -EAGAIN;
2076
2077         if (seqno == ring->outstanding_lazy_request) {
2078                 struct drm_i915_gem_request *request;
2079
2080                 request = kzalloc(sizeof(*request), GFP_KERNEL);
2081                 if (request == NULL)
2082                         return -ENOMEM;
2083
2084                 ret = i915_add_request(dev, NULL, request, ring);
2085                 if (ret) {
2086                         kfree(request);
2087                         return ret;
2088                 }
2089
2090                 seqno = request->seqno;
2091         }
2092
2093         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2094                 if (HAS_PCH_SPLIT(dev))
2095                         ier = I915_READ(DEIER) | I915_READ(GTIER);
2096                 else
2097                         ier = I915_READ(IER);
2098                 if (!ier) {
2099                         DRM_ERROR("something (likely vbetool) disabled "
2100                                   "interrupts, re-enabling\n");
2101                         i915_driver_irq_preinstall(dev);
2102                         i915_driver_irq_postinstall(dev);
2103                 }
2104
2105                 trace_i915_gem_request_wait_begin(dev, seqno);
2106
2107                 ring->waiting_seqno = seqno;
2108                 ring->user_irq_get(ring);
2109                 if (interruptible)
2110                         ret = wait_event_interruptible(ring->irq_queue,
2111                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2112                                 || atomic_read(&dev_priv->mm.wedged));
2113                 else
2114                         wait_event(ring->irq_queue,
2115                                 i915_seqno_passed(ring->get_seqno(ring), seqno)
2116                                 || atomic_read(&dev_priv->mm.wedged));
2117
2118                 ring->user_irq_put(ring);
2119                 ring->waiting_seqno = 0;
2120
2121                 trace_i915_gem_request_wait_end(dev, seqno);
2122         }
2123         if (atomic_read(&dev_priv->mm.wedged))
2124                 ret = -EAGAIN;
2125
2126         if (ret && ret != -ERESTARTSYS)
2127                 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2128                           __func__, ret, seqno, ring->get_seqno(ring),
2129                           dev_priv->next_seqno);
2130
2131         /* Directly dispatch request retiring.  While we have the work queue
2132          * to handle this, the waiter on a request often wants an associated
2133          * buffer to have made it to the inactive list, and we would need
2134          * a separate wait queue to handle that.
2135          */
2136         if (ret == 0)
2137                 i915_gem_retire_requests_ring(dev, ring);
2138
2139         return ret;
2140 }
2141
2142 /**
2143  * Waits for a sequence number to be signaled, and cleans up the
2144  * request and object lists appropriately for that event.
2145  */
2146 static int
2147 i915_wait_request(struct drm_device *dev, uint32_t seqno,
2148                   struct intel_ring_buffer *ring)
2149 {
2150         return i915_do_wait_request(dev, seqno, 1, ring);
2151 }
2152
2153 static void
2154 i915_gem_flush_ring(struct drm_device *dev,
2155                     struct intel_ring_buffer *ring,
2156                     uint32_t invalidate_domains,
2157                     uint32_t flush_domains)
2158 {
2159         ring->flush(ring, invalidate_domains, flush_domains);
2160         i915_gem_process_flushing_list(dev, flush_domains, ring);
2161 }
2162
2163 static void
2164 i915_gem_flush(struct drm_device *dev,
2165                uint32_t invalidate_domains,
2166                uint32_t flush_domains,
2167                uint32_t flush_rings)
2168 {
2169         drm_i915_private_t *dev_priv = dev->dev_private;
2170
2171         if (flush_domains & I915_GEM_DOMAIN_CPU)
2172                 intel_gtt_chipset_flush();
2173
2174         if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2175                 if (flush_rings & RING_RENDER)
2176                         i915_gem_flush_ring(dev, &dev_priv->render_ring,
2177                                             invalidate_domains, flush_domains);
2178                 if (flush_rings & RING_BSD)
2179                         i915_gem_flush_ring(dev, &dev_priv->bsd_ring,
2180                                             invalidate_domains, flush_domains);
2181                 if (flush_rings & RING_BLT)
2182                         i915_gem_flush_ring(dev, &dev_priv->blt_ring,
2183                                             invalidate_domains, flush_domains);
2184         }
2185 }
2186
2187 /**
2188  * Ensures that all rendering to the object has completed and the object is
2189  * safe to unbind from the GTT or access from the CPU.
2190  */
2191 static int
2192 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
2193                                bool interruptible)
2194 {
2195         struct drm_device *dev = obj->base.dev;
2196         int ret;
2197
2198         /* This function only exists to support waiting for existing rendering,
2199          * not for emitting required flushes.
2200          */
2201         BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2202
2203         /* If there is rendering queued on the buffer being evicted, wait for
2204          * it.
2205          */
2206         if (obj->active) {
2207                 ret = i915_do_wait_request(dev,
2208                                            obj->last_rendering_seqno,
2209                                            interruptible,
2210                                            obj->ring);
2211                 if (ret)
2212                         return ret;
2213         }
2214
2215         return 0;
2216 }
2217
2218 /**
2219  * Unbinds an object from the GTT aperture.
2220  */
2221 int
2222 i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2223 {
2224         struct drm_device *dev = obj->base.dev;
2225         struct drm_i915_private *dev_priv = dev->dev_private;
2226         int ret = 0;
2227
2228         if (obj->gtt_space == NULL)
2229                 return 0;
2230
2231         if (obj->pin_count != 0) {
2232                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2233                 return -EINVAL;
2234         }
2235
2236         /* blow away mappings if mapped through GTT */
2237         i915_gem_release_mmap(obj);
2238
2239         /* Move the object to the CPU domain to ensure that
2240          * any possible CPU writes while it's not in the GTT
2241          * are flushed when we go to remap it. This will
2242          * also ensure that all pending GPU writes are finished
2243          * before we unbind.
2244          */
2245         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2246         if (ret == -ERESTARTSYS)
2247                 return ret;
2248         /* Continue on if we fail due to EIO, the GPU is hung so we
2249          * should be safe and we need to cleanup or else we might
2250          * cause memory corruption through use-after-free.
2251          */
2252         if (ret) {
2253                 i915_gem_clflush_object(obj);
2254                 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2255         }
2256
2257         /* release the fence reg _after_ flushing */
2258         if (obj->fence_reg != I915_FENCE_REG_NONE)
2259                 i915_gem_clear_fence_reg(obj);
2260
2261         i915_gem_gtt_unbind_object(obj);
2262
2263         i915_gem_object_put_pages_gtt(obj);
2264
2265         i915_gem_info_remove_gtt(dev_priv, obj);
2266         list_del_init(&obj->mm_list);
2267         /* Avoid an unnecessary call to unbind on rebind. */
2268         obj->map_and_fenceable = true;
2269
2270         drm_mm_put_block(obj->gtt_space);
2271         obj->gtt_space = NULL;
2272         obj->gtt_offset = 0;
2273
2274         if (i915_gem_object_is_purgeable(obj))
2275                 i915_gem_object_truncate(obj);
2276
2277         trace_i915_gem_object_unbind(obj);
2278
2279         return ret;
2280 }
2281
2282 static int i915_ring_idle(struct drm_device *dev,
2283                           struct intel_ring_buffer *ring)
2284 {
2285         if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2286                 return 0;
2287
2288         i915_gem_flush_ring(dev, ring,
2289                             I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2290         return i915_wait_request(dev,
2291                                  i915_gem_next_request_seqno(dev, ring),
2292                                  ring);
2293 }
2294
2295 int
2296 i915_gpu_idle(struct drm_device *dev)
2297 {
2298         drm_i915_private_t *dev_priv = dev->dev_private;
2299         bool lists_empty;
2300         int ret;
2301
2302         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2303                        list_empty(&dev_priv->mm.active_list));
2304         if (lists_empty)
2305                 return 0;
2306
2307         /* Flush everything onto the inactive list. */
2308         ret = i915_ring_idle(dev, &dev_priv->render_ring);
2309         if (ret)
2310                 return ret;
2311
2312         ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
2313         if (ret)
2314                 return ret;
2315
2316         ret = i915_ring_idle(dev, &dev_priv->blt_ring);
2317         if (ret)
2318                 return ret;
2319
2320         return 0;
2321 }
2322
2323 static void sandybridge_write_fence_reg(struct drm_i915_gem_object *obj)
2324 {
2325         struct drm_device *dev = obj->base.dev;
2326         drm_i915_private_t *dev_priv = dev->dev_private;
2327         u32 size = obj->gtt_space->size;
2328         int regnum = obj->fence_reg;
2329         uint64_t val;
2330
2331         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2332                     0xfffff000) << 32;
2333         val |= obj->gtt_offset & 0xfffff000;
2334         val |= (uint64_t)((obj->stride / 128) - 1) <<
2335                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2336
2337         if (obj->tiling_mode == I915_TILING_Y)
2338                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2339         val |= I965_FENCE_REG_VALID;
2340
2341         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2342 }
2343
2344 static void i965_write_fence_reg(struct drm_i915_gem_object *obj)
2345 {
2346         struct drm_device *dev = obj->base.dev;
2347         drm_i915_private_t *dev_priv = dev->dev_private;
2348         u32 size = obj->gtt_space->size;
2349         int regnum = obj->fence_reg;
2350         uint64_t val;
2351
2352         val = (uint64_t)((obj->gtt_offset + size - 4096) &
2353                     0xfffff000) << 32;
2354         val |= obj->gtt_offset & 0xfffff000;
2355         val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2356         if (obj->tiling_mode == I915_TILING_Y)
2357                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2358         val |= I965_FENCE_REG_VALID;
2359
2360         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2361 }
2362
2363 static void i915_write_fence_reg(struct drm_i915_gem_object *obj)
2364 {
2365         struct drm_device *dev = obj->base.dev;
2366         drm_i915_private_t *dev_priv = dev->dev_private;
2367         u32 size = obj->gtt_space->size;
2368         uint32_t fence_reg, val, pitch_val;
2369         int tile_width;
2370
2371         if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2372             (obj->gtt_offset & (size - 1))) {
2373                 WARN(1, "%s: object 0x%08x [fenceable? %d] not 1M or size (0x%08x) aligned [gtt_space offset=%lx, size=%lx]\n",
2374                      __func__, obj->gtt_offset, obj->map_and_fenceable, size,
2375                      obj->gtt_space->start, obj->gtt_space->size);
2376                 return;
2377         }
2378
2379         if (obj->tiling_mode == I915_TILING_Y &&
2380             HAS_128_BYTE_Y_TILING(dev))
2381                 tile_width = 128;
2382         else
2383                 tile_width = 512;
2384
2385         /* Note: pitch better be a power of two tile widths */
2386         pitch_val = obj->stride / tile_width;
2387         pitch_val = ffs(pitch_val) - 1;
2388
2389         if (obj->tiling_mode == I915_TILING_Y &&
2390             HAS_128_BYTE_Y_TILING(dev))
2391                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2392         else
2393                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2394
2395         val = obj->gtt_offset;
2396         if (obj->tiling_mode == I915_TILING_Y)
2397                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2398         val |= I915_FENCE_SIZE_BITS(size);
2399         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2400         val |= I830_FENCE_REG_VALID;
2401
2402         fence_reg = obj->fence_reg;
2403         if (fence_reg < 8)
2404                 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2405         else
2406                 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2407         I915_WRITE(fence_reg, val);
2408 }
2409
2410 static void i830_write_fence_reg(struct drm_i915_gem_object *obj)
2411 {
2412         struct drm_device *dev = obj->base.dev;
2413         drm_i915_private_t *dev_priv = dev->dev_private;
2414         u32 size = obj->gtt_space->size;
2415         int regnum = obj->fence_reg;
2416         uint32_t val;
2417         uint32_t pitch_val;
2418         uint32_t fence_size_bits;
2419
2420         if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2421             (obj->gtt_offset & (obj->base.size - 1))) {
2422                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2423                      __func__, obj->gtt_offset);
2424                 return;
2425         }
2426
2427         pitch_val = obj->stride / 128;
2428         pitch_val = ffs(pitch_val) - 1;
2429         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2430
2431         val = obj->gtt_offset;
2432         if (obj->tiling_mode == I915_TILING_Y)
2433                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2434         fence_size_bits = I830_FENCE_SIZE_BITS(size);
2435         WARN_ON(fence_size_bits & ~0x00000f00);
2436         val |= fence_size_bits;
2437         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2438         val |= I830_FENCE_REG_VALID;
2439
2440         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2441 }
2442
2443 static int i915_find_fence_reg(struct drm_device *dev,
2444                                bool interruptible)
2445 {
2446         struct drm_i915_private *dev_priv = dev->dev_private;
2447         struct drm_i915_fence_reg *reg;
2448         struct drm_i915_gem_object *obj = NULL;
2449         int i, avail, ret;
2450
2451         /* First try to find a free reg */
2452         avail = 0;
2453         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2454                 reg = &dev_priv->fence_regs[i];
2455                 if (!reg->obj)
2456                         return i;
2457
2458                 if (!reg->obj->pin_count)
2459                         avail++;
2460         }
2461
2462         if (avail == 0)
2463                 return -ENOSPC;
2464
2465         /* None available, try to steal one or wait for a user to finish */
2466         avail = I915_FENCE_REG_NONE;
2467         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2468                             lru_list) {
2469                 obj = reg->obj;
2470                 if (obj->pin_count)
2471                         continue;
2472
2473                 /* found one! */
2474                 avail = obj->fence_reg;
2475                 break;
2476         }
2477
2478         BUG_ON(avail == I915_FENCE_REG_NONE);
2479
2480         /* We only have a reference on obj from the active list. put_fence_reg
2481          * might drop that one, causing a use-after-free in it. So hold a
2482          * private reference to obj like the other callers of put_fence_reg
2483          * (set_tiling ioctl) do. */
2484         drm_gem_object_reference(&obj->base);
2485         ret = i915_gem_object_put_fence_reg(obj, interruptible);
2486         drm_gem_object_unreference(&obj->base);
2487         if (ret != 0)
2488                 return ret;
2489
2490         return avail;
2491 }
2492
2493 /**
2494  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2495  * @obj: object to map through a fence reg
2496  *
2497  * When mapping objects through the GTT, userspace wants to be able to write
2498  * to them without having to worry about swizzling if the object is tiled.
2499  *
2500  * This function walks the fence regs looking for a free one for @obj,
2501  * stealing one if it can't find any.
2502  *
2503  * It then sets up the reg based on the object's properties: address, pitch
2504  * and tiling format.
2505  */
2506 int
2507 i915_gem_object_get_fence_reg(struct drm_i915_gem_object *obj,
2508                               bool interruptible)
2509 {
2510         struct drm_device *dev = obj->base.dev;
2511         struct drm_i915_private *dev_priv = dev->dev_private;
2512         struct drm_i915_fence_reg *reg = NULL;
2513         int ret;
2514
2515         /* Just update our place in the LRU if our fence is getting used. */
2516         if (obj->fence_reg != I915_FENCE_REG_NONE) {
2517                 reg = &dev_priv->fence_regs[obj->fence_reg];
2518                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2519                 return 0;
2520         }
2521
2522         switch (obj->tiling_mode) {
2523         case I915_TILING_NONE:
2524                 WARN(1, "allocating a fence for non-tiled object?\n");
2525                 break;
2526         case I915_TILING_X:
2527                 if (!obj->stride)
2528                         return -EINVAL;
2529                 WARN((obj->stride & (512 - 1)),
2530                      "object 0x%08x is X tiled but has non-512B pitch\n",
2531                      obj->gtt_offset);
2532                 break;
2533         case I915_TILING_Y:
2534                 if (!obj->stride)
2535                         return -EINVAL;
2536                 WARN((obj->stride & (128 - 1)),
2537                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2538                      obj->gtt_offset);
2539                 break;
2540         }
2541
2542         ret = i915_find_fence_reg(dev, interruptible);
2543         if (ret < 0)
2544                 return ret;
2545
2546         obj->fence_reg = ret;
2547         reg = &dev_priv->fence_regs[obj->fence_reg];
2548         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2549
2550         reg->obj = obj;
2551
2552         switch (INTEL_INFO(dev)->gen) {
2553         case 6:
2554                 sandybridge_write_fence_reg(obj);
2555                 break;
2556         case 5:
2557         case 4:
2558                 i965_write_fence_reg(obj);
2559                 break;
2560         case 3:
2561                 i915_write_fence_reg(obj);
2562                 break;
2563         case 2:
2564                 i830_write_fence_reg(obj);
2565                 break;
2566         }
2567
2568         trace_i915_gem_object_get_fence(obj,
2569                                         obj->fence_reg,
2570                                         obj->tiling_mode);
2571
2572         return 0;
2573 }
2574
2575 /**
2576  * i915_gem_clear_fence_reg - clear out fence register info
2577  * @obj: object to clear
2578  *
2579  * Zeroes out the fence register itself and clears out the associated
2580  * data structures in dev_priv and obj.
2581  */
2582 static void
2583 i915_gem_clear_fence_reg(struct drm_i915_gem_object *obj)
2584 {
2585         struct drm_device *dev = obj->base.dev;
2586         drm_i915_private_t *dev_priv = dev->dev_private;
2587         struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[obj->fence_reg];
2588         uint32_t fence_reg;
2589
2590         switch (INTEL_INFO(dev)->gen) {
2591         case 6:
2592                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2593                              (obj->fence_reg * 8), 0);
2594                 break;
2595         case 5:
2596         case 4:
2597                 I915_WRITE64(FENCE_REG_965_0 + (obj->fence_reg * 8), 0);
2598                 break;
2599         case 3:
2600                 if (obj->fence_reg >= 8)
2601                         fence_reg = FENCE_REG_945_8 + (obj->fence_reg - 8) * 4;
2602                 else
2603         case 2:
2604                         fence_reg = FENCE_REG_830_0 + obj->fence_reg * 4;
2605
2606                 I915_WRITE(fence_reg, 0);
2607                 break;
2608         }
2609
2610         reg->obj = NULL;
2611         obj->fence_reg = I915_FENCE_REG_NONE;
2612         list_del_init(&reg->lru_list);
2613 }
2614
2615 /**
2616  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2617  * to the buffer to finish, and then resets the fence register.
2618  * @obj: tiled object holding a fence register.
2619  * @bool: whether the wait upon the fence is interruptible
2620  *
2621  * Zeroes out the fence register itself and clears out the associated
2622  * data structures in dev_priv and obj.
2623  */
2624 int
2625 i915_gem_object_put_fence_reg(struct drm_i915_gem_object *obj,
2626                               bool interruptible)
2627 {
2628         struct drm_device *dev = obj->base.dev;
2629         struct drm_i915_private *dev_priv = dev->dev_private;
2630         struct drm_i915_fence_reg *reg;
2631
2632         if (obj->fence_reg == I915_FENCE_REG_NONE)
2633                 return 0;
2634
2635         /* If we've changed tiling, GTT-mappings of the object
2636          * need to re-fault to ensure that the correct fence register
2637          * setup is in place.
2638          */
2639         i915_gem_release_mmap(obj);
2640
2641         /* On the i915, GPU access to tiled buffers is via a fence,
2642          * therefore we must wait for any outstanding access to complete
2643          * before clearing the fence.
2644          */
2645         reg = &dev_priv->fence_regs[obj->fence_reg];
2646         if (reg->gpu) {
2647                 int ret;
2648
2649                 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2650                 if (ret)
2651                         return ret;
2652
2653                 ret = i915_gem_object_wait_rendering(obj, interruptible);
2654                 if (ret)
2655                         return ret;
2656
2657                 reg->gpu = false;
2658         }
2659
2660         i915_gem_object_flush_gtt_write_domain(obj);
2661         i915_gem_clear_fence_reg(obj);
2662
2663         return 0;
2664 }
2665
2666 /**
2667  * Finds free space in the GTT aperture and binds the object there.
2668  */
2669 static int
2670 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2671                             unsigned alignment,
2672                             bool map_and_fenceable)
2673 {
2674         struct drm_device *dev = obj->base.dev;
2675         drm_i915_private_t *dev_priv = dev->dev_private;
2676         struct drm_mm_node *free_space;
2677         gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2678         u32 size, fence_size, fence_alignment, unfenced_alignment;
2679         bool mappable, fenceable;
2680         int ret;
2681
2682         if (obj->madv != I915_MADV_WILLNEED) {
2683                 DRM_ERROR("Attempting to bind a purgeable object\n");
2684                 return -EINVAL;
2685         }
2686
2687         fence_size = i915_gem_get_gtt_size(obj);
2688         fence_alignment = i915_gem_get_gtt_alignment(obj);
2689         unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj);
2690
2691         if (alignment == 0)
2692                 alignment = map_and_fenceable ? fence_alignment :
2693                                                 unfenced_alignment;
2694         if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2695                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2696                 return -EINVAL;
2697         }
2698
2699         size = map_and_fenceable ? fence_size : obj->base.size;
2700
2701         /* If the object is bigger than the entire aperture, reject it early
2702          * before evicting everything in a vain attempt to find space.
2703          */
2704         if (obj->base.size >
2705             (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2706                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2707                 return -E2BIG;
2708         }
2709
2710  search_free:
2711         if (map_and_fenceable)
2712                 free_space =
2713                         drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2714                                                     size, alignment, 0,
2715                                                     dev_priv->mm.gtt_mappable_end,
2716                                                     0);
2717         else
2718                 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2719                                                 size, alignment, 0);
2720
2721         if (free_space != NULL) {
2722                 if (map_and_fenceable)
2723                         obj->gtt_space =
2724                                 drm_mm_get_block_range_generic(free_space,
2725                                                                size, alignment, 0,
2726                                                                dev_priv->mm.gtt_mappable_end,
2727                                                                0);
2728                 else
2729                         obj->gtt_space =
2730                                 drm_mm_get_block(free_space, size, alignment);
2731         }
2732         if (obj->gtt_space == NULL) {
2733                 /* If the gtt is empty and we're still having trouble
2734                  * fitting our object in, we're out of memory.
2735                  */
2736                 ret = i915_gem_evict_something(dev, size, alignment,
2737                                                map_and_fenceable);
2738                 if (ret)
2739                         return ret;
2740
2741                 goto search_free;
2742         }
2743
2744         ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2745         if (ret) {
2746                 drm_mm_put_block(obj->gtt_space);
2747                 obj->gtt_space = NULL;
2748
2749                 if (ret == -ENOMEM) {
2750                         /* first try to clear up some space from the GTT */
2751                         ret = i915_gem_evict_something(dev, size,
2752                                                        alignment,
2753                                                        map_and_fenceable);
2754                         if (ret) {
2755                                 /* now try to shrink everyone else */
2756                                 if (gfpmask) {
2757                                         gfpmask = 0;
2758                                         goto search_free;
2759                                 }
2760
2761                                 return ret;
2762                         }
2763
2764                         goto search_free;
2765                 }
2766
2767                 return ret;
2768         }
2769
2770         ret = i915_gem_gtt_bind_object(obj);
2771         if (ret) {
2772                 i915_gem_object_put_pages_gtt(obj);
2773                 drm_mm_put_block(obj->gtt_space);
2774                 obj->gtt_space = NULL;
2775
2776                 ret = i915_gem_evict_something(dev, size,
2777                                                alignment, map_and_fenceable);
2778                 if (ret)
2779                         return ret;
2780
2781                 goto search_free;
2782         }
2783
2784         obj->gtt_offset = obj->gtt_space->start;
2785
2786         /* keep track of bounds object by adding it to the inactive list */
2787         list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2788         i915_gem_info_add_gtt(dev_priv, obj);
2789
2790         /* Assert that the object is not currently in any GPU domain. As it
2791          * wasn't in the GTT, there shouldn't be any way it could have been in
2792          * a GPU cache
2793          */
2794         BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2795         BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2796
2797         trace_i915_gem_object_bind(obj, obj->gtt_offset, map_and_fenceable);
2798
2799         fenceable =
2800                 obj->gtt_space->size == fence_size &&
2801                 (obj->gtt_space->start & (fence_alignment -1)) == 0;
2802
2803         mappable =
2804                 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2805
2806         obj->map_and_fenceable = mappable && fenceable;
2807
2808         return 0;
2809 }
2810
2811 void
2812 i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2813 {
2814         /* If we don't have a page list set up, then we're not pinned
2815          * to GPU, and we can ignore the cache flush because it'll happen
2816          * again at bind time.
2817          */
2818         if (obj->pages == NULL)
2819                 return;
2820
2821         trace_i915_gem_object_clflush(obj);
2822
2823         drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2824 }
2825
2826 /** Flushes any GPU write domain for the object if it's dirty. */
2827 static int
2828 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj,
2829                                        bool pipelined)
2830 {
2831         struct drm_device *dev = obj->base.dev;
2832
2833         if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2834                 return 0;
2835
2836         /* Queue the GPU write cache flushing we need. */
2837         i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
2838         BUG_ON(obj->base.write_domain);
2839
2840         if (pipelined)
2841                 return 0;
2842
2843         return i915_gem_object_wait_rendering(obj, true);
2844 }
2845
2846 /** Flushes the GTT write domain for the object if it's dirty. */
2847 static void
2848 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2849 {
2850         uint32_t old_write_domain;
2851
2852         if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2853                 return;
2854
2855         /* No actual flushing is required for the GTT write domain.   Writes
2856          * to it immediately go to main memory as far as we know, so there's
2857          * no chipset flush.  It also doesn't land in render cache.
2858          */
2859         i915_gem_release_mmap(obj);
2860
2861         old_write_domain = obj->base.write_domain;
2862         obj->base.write_domain = 0;
2863
2864         trace_i915_gem_object_change_domain(obj,
2865                                             obj->base.read_domains,
2866                                             old_write_domain);
2867 }
2868
2869 /** Flushes the CPU write domain for the object if it's dirty. */
2870 static void
2871 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2872 {
2873         uint32_t old_write_domain;
2874
2875         if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2876                 return;
2877
2878         i915_gem_clflush_object(obj);
2879         intel_gtt_chipset_flush();
2880         old_write_domain = obj->base.write_domain;
2881         obj->base.write_domain = 0;
2882
2883         trace_i915_gem_object_change_domain(obj,
2884                                             obj->base.read_domains,
2885                                             old_write_domain);
2886 }
2887
2888 /**
2889  * Moves a single object to the GTT read, and possibly write domain.
2890  *
2891  * This function returns when the move is complete, including waiting on
2892  * flushes to occur.
2893  */
2894 int
2895 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, int write)
2896 {
2897         uint32_t old_write_domain, old_read_domains;
2898         int ret;
2899
2900         /* Not valid to be called on unbound objects. */
2901         if (obj->gtt_space == NULL)
2902                 return -EINVAL;
2903
2904         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
2905         if (ret != 0)
2906                 return ret;
2907
2908         i915_gem_object_flush_cpu_write_domain(obj);
2909
2910         if (write) {
2911                 ret = i915_gem_object_wait_rendering(obj, true);
2912                 if (ret)
2913                         return ret;
2914         }
2915
2916         old_write_domain = obj->base.write_domain;
2917         old_read_domains = obj->base.read_domains;
2918
2919         /* It should now be out of any other write domains, and we can update
2920          * the domain values for our changes.
2921          */
2922         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2923         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2924         if (write) {
2925                 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2926                 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2927                 obj->dirty = 1;
2928         }
2929
2930         trace_i915_gem_object_change_domain(obj,
2931                                             old_read_domains,
2932                                             old_write_domain);
2933
2934         return 0;
2935 }
2936
2937 /*
2938  * Prepare buffer for display plane. Use uninterruptible for possible flush
2939  * wait, as in modesetting process we're not supposed to be interrupted.
2940  */
2941 int
2942 i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj,
2943                                      bool pipelined)
2944 {
2945         uint32_t old_read_domains;
2946         int ret;
2947
2948         /* Not valid to be called on unbound objects. */
2949         if (obj->gtt_space == NULL)
2950                 return -EINVAL;
2951
2952         ret = i915_gem_object_flush_gpu_write_domain(obj, true);
2953         if (ret)
2954                 return ret;
2955
2956         /* Currently, we are always called from an non-interruptible context. */
2957         if (!pipelined) {
2958                 ret = i915_gem_object_wait_rendering(obj, false);
2959                 if (ret)
2960                         return ret;
2961         }
2962
2963         i915_gem_object_flush_cpu_write_domain(obj);
2964
2965         old_read_domains = obj->base.read_domains;
2966         obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2967
2968         trace_i915_gem_object_change_domain(obj,
2969                                             old_read_domains,
2970                                             obj->base.write_domain);
2971
2972         return 0;
2973 }
2974
2975 int
2976 i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
2977                           bool interruptible)
2978 {
2979         if (!obj->active)
2980                 return 0;
2981
2982         if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
2983                 i915_gem_flush_ring(obj->base.dev, obj->ring,
2984                                     0, obj->base.write_domain);
2985
2986         return i915_gem_object_wait_rendering(obj, interruptible);
2987 }
2988
2989 /**
2990  * Moves a single object to the CPU read, and possibly write domain.
2991  *
2992  * This function returns when the move is complete, including waiting on
2993  * flushes to occur.
2994  */
2995 static int
2996 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, int write)
2997 {
2998         uint32_t old_write_domain, old_read_domains;
2999         int ret;
3000
3001         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3002         if (ret != 0)
3003                 return ret;
3004
3005         i915_gem_object_flush_gtt_write_domain(obj);
3006
3007         /* If we have a partially-valid cache of the object in the CPU,
3008          * finish invalidating it and free the per-page flags.
3009          */
3010         i915_gem_object_set_to_full_cpu_read_domain(obj);
3011
3012         if (write) {
3013                 ret = i915_gem_object_wait_rendering(obj, true);
3014                 if (ret)
3015                         return ret;
3016         }
3017
3018         old_write_domain = obj->base.write_domain;
3019         old_read_domains = obj->base.read_domains;
3020
3021         /* Flush the CPU cache if it's still invalid. */
3022         if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3023                 i915_gem_clflush_object(obj);
3024
3025                 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3026         }
3027
3028         /* It should now be out of any other write domains, and we can update
3029          * the domain values for our changes.
3030          */
3031         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3032
3033         /* If we're writing through the CPU, then the GPU read domains will
3034          * need to be invalidated at next use.
3035          */
3036         if (write) {
3037                 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3038                 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3039         }
3040
3041         trace_i915_gem_object_change_domain(obj,
3042                                             old_read_domains,
3043                                             old_write_domain);
3044
3045         return 0;
3046 }
3047
3048 /*
3049  * Set the next domain for the specified object. This
3050  * may not actually perform the necessary flushing/invaliding though,
3051  * as that may want to be batched with other set_domain operations
3052  *
3053  * This is (we hope) the only really tricky part of gem. The goal
3054  * is fairly simple -- track which caches hold bits of the object
3055  * and make sure they remain coherent. A few concrete examples may
3056  * help to explain how it works. For shorthand, we use the notation
3057  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3058  * a pair of read and write domain masks.
3059  *
3060  * Case 1: the batch buffer
3061  *
3062  *      1. Allocated
3063  *      2. Written by CPU
3064  *      3. Mapped to GTT
3065  *      4. Read by GPU
3066  *      5. Unmapped from GTT
3067  *      6. Freed
3068  *
3069  *      Let's take these a step at a time
3070  *
3071  *      1. Allocated
3072  *              Pages allocated from the kernel may still have
3073  *              cache contents, so we set them to (CPU, CPU) always.
3074  *      2. Written by CPU (using pwrite)
3075  *              The pwrite function calls set_domain (CPU, CPU) and
3076  *              this function does nothing (as nothing changes)
3077  *      3. Mapped by GTT
3078  *              This function asserts that the object is not
3079  *              currently in any GPU-based read or write domains
3080  *      4. Read by GPU
3081  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3082  *              As write_domain is zero, this function adds in the
3083  *              current read domains (CPU+COMMAND, 0).
3084  *              flush_domains is set to CPU.
3085  *              invalidate_domains is set to COMMAND
3086  *              clflush is run to get data out of the CPU caches
3087  *              then i915_dev_set_domain calls i915_gem_flush to
3088  *              emit an MI_FLUSH and drm_agp_chipset_flush
3089  *      5. Unmapped from GTT
3090  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3091  *              flush_domains and invalidate_domains end up both zero
3092  *              so no flushing/invalidating happens
3093  *      6. Freed
3094  *              yay, done
3095  *
3096  * Case 2: The shared render buffer
3097  *
3098  *      1. Allocated
3099  *      2. Mapped to GTT
3100  *      3. Read/written by GPU
3101  *      4. set_domain to (CPU,CPU)
3102  *      5. Read/written by CPU
3103  *      6. Read/written by GPU
3104  *
3105  *      1. Allocated
3106  *              Same as last example, (CPU, CPU)
3107  *      2. Mapped to GTT
3108  *              Nothing changes (assertions find that it is not in the GPU)
3109  *      3. Read/written by GPU
3110  *              execbuffer calls set_domain (RENDER, RENDER)
3111  *              flush_domains gets CPU
3112  *              invalidate_domains gets GPU
3113  *              clflush (obj)
3114  *              MI_FLUSH and drm_agp_chipset_flush
3115  *      4. set_domain (CPU, CPU)
3116  *              flush_domains gets GPU
3117  *              invalidate_domains gets CPU
3118  *              wait_rendering (obj) to make sure all drawing is complete.
3119  *              This will include an MI_FLUSH to get the data from GPU
3120  *              to memory
3121  *              clflush (obj) to invalidate the CPU cache
3122  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3123  *      5. Read/written by CPU
3124  *              cache lines are loaded and dirtied
3125  *      6. Read written by GPU
3126  *              Same as last GPU access
3127  *
3128  * Case 3: The constant buffer
3129  *
3130  *      1. Allocated
3131  *      2. Written by CPU
3132  *      3. Read by GPU
3133  *      4. Updated (written) by CPU again
3134  *      5. Read by GPU
3135  *
3136  *      1. Allocated
3137  *              (CPU, CPU)
3138  *      2. Written by CPU
3139  *              (CPU, CPU)
3140  *      3. Read by GPU
3141  *              (CPU+RENDER, 0)
3142  *              flush_domains = CPU
3143  *              invalidate_domains = RENDER
3144  *              clflush (obj)
3145  *              MI_FLUSH
3146  *              drm_agp_chipset_flush
3147  *      4. Updated (written) by CPU again
3148  *              (CPU, CPU)
3149  *              flush_domains = 0 (no previous write domain)
3150  *              invalidate_domains = 0 (no new read domains)
3151  *      5. Read by GPU
3152  *              (CPU+RENDER, 0)
3153  *              flush_domains = CPU
3154  *              invalidate_domains = RENDER
3155  *              clflush (obj)
3156  *              MI_FLUSH
3157  *              drm_agp_chipset_flush
3158  */
3159 static void
3160 i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj,
3161                                   struct intel_ring_buffer *ring,
3162                                   struct change_domains *cd)
3163 {
3164         uint32_t invalidate_domains = 0, flush_domains = 0;
3165
3166         /*
3167          * If the object isn't moving to a new write domain,
3168          * let the object stay in multiple read domains
3169          */
3170         if (obj->base.pending_write_domain == 0)
3171                 obj->base.pending_read_domains |= obj->base.read_domains;
3172
3173         /*
3174          * Flush the current write domain if
3175          * the new read domains don't match. Invalidate
3176          * any read domains which differ from the old
3177          * write domain
3178          */
3179         if (obj->base.write_domain &&
3180             (obj->base.write_domain != obj->base.pending_read_domains ||
3181              obj->ring != ring)) {
3182                 flush_domains |= obj->base.write_domain;
3183                 invalidate_domains |=
3184                         obj->base.pending_read_domains & ~obj->base.write_domain;
3185         }
3186         /*
3187          * Invalidate any read caches which may have
3188          * stale data. That is, any new read domains.
3189          */
3190         invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains;
3191         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
3192                 i915_gem_clflush_object(obj);
3193
3194         /* blow away mappings if mapped through GTT */
3195         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT)
3196                 i915_gem_release_mmap(obj);
3197
3198         /* The actual obj->write_domain will be updated with
3199          * pending_write_domain after we emit the accumulated flush for all
3200          * of our domain changes in execbuffers (which clears objects'
3201          * write_domains).  So if we have a current write domain that we
3202          * aren't changing, set pending_write_domain to that.
3203          */
3204         if (flush_domains == 0 && obj->base.pending_write_domain == 0)
3205                 obj->base.pending_write_domain = obj->base.write_domain;
3206
3207         cd->invalidate_domains |= invalidate_domains;
3208         cd->flush_domains |= flush_domains;
3209         if (flush_domains & I915_GEM_GPU_DOMAINS)
3210                 cd->flush_rings |= obj->ring->id;
3211         if (invalidate_domains & I915_GEM_GPU_DOMAINS)
3212                 cd->flush_rings |= ring->id;
3213 }
3214
3215 /**
3216  * Moves the object from a partially CPU read to a full one.
3217  *
3218  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3219  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3220  */
3221 static void
3222 i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
3223 {
3224         if (!obj->page_cpu_valid)
3225                 return;
3226
3227         /* If we're partially in the CPU read domain, finish moving it in.
3228          */
3229         if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
3230                 int i;
3231
3232                 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3233                         if (obj->page_cpu_valid[i])
3234                                 continue;
3235                         drm_clflush_pages(obj->pages + i, 1);
3236                 }
3237         }
3238
3239         /* Free the page_cpu_valid mappings which are now stale, whether
3240          * or not we've got I915_GEM_DOMAIN_CPU.
3241          */
3242         kfree(obj->page_cpu_valid);
3243         obj->page_cpu_valid = NULL;
3244 }
3245
3246 /**
3247  * Set the CPU read domain on a range of the object.
3248  *
3249  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3250  * not entirely valid.  The page_cpu_valid member of the object flags which
3251  * pages have been flushed, and will be respected by
3252  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3253  * of the whole object.
3254  *
3255  * This function returns when the move is complete, including waiting on
3256  * flushes to occur.
3257  */
3258 static int
3259 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3260                                           uint64_t offset, uint64_t size)
3261 {
3262         uint32_t old_read_domains;
3263         int i, ret;
3264
3265         if (offset == 0 && size == obj->base.size)
3266                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3267
3268         ret = i915_gem_object_flush_gpu_write_domain(obj, false);
3269         if (ret != 0)
3270                 return ret;
3271         i915_gem_object_flush_gtt_write_domain(obj);
3272
3273         /* If we're already fully in the CPU read domain, we're done. */
3274         if (obj->page_cpu_valid == NULL &&
3275             (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3276                 return 0;
3277
3278         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3279          * newly adding I915_GEM_DOMAIN_CPU
3280          */
3281         if (obj->page_cpu_valid == NULL) {
3282                 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3283                                               GFP_KERNEL);
3284                 if (obj->page_cpu_valid == NULL)
3285                         return -ENOMEM;
3286         } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3287                 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3288
3289         /* Flush the cache on any pages that are still invalid from the CPU's
3290          * perspective.
3291          */
3292         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3293              i++) {
3294                 if (obj->page_cpu_valid[i])
3295                         continue;
3296
3297                 drm_clflush_pages(obj->pages + i, 1);
3298
3299                 obj->page_cpu_valid[i] = 1;
3300         }
3301
3302         /* It should now be out of any other write domains, and we can update
3303          * the domain values for our changes.
3304          */
3305         BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3306
3307         old_read_domains = obj->base.read_domains;
3308         obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3309
3310         trace_i915_gem_object_change_domain(obj,
3311                                             old_read_domains,
3312                                             obj->base.write_domain);
3313
3314         return 0;
3315 }
3316
3317 static int
3318 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
3319                                    struct drm_file *file_priv,
3320                                    struct drm_i915_gem_exec_object2 *entry,
3321                                    struct drm_i915_gem_relocation_entry *reloc)
3322 {
3323         struct drm_device *dev = obj->base.dev;
3324         struct drm_gem_object *target_obj;
3325         uint32_t target_offset;
3326         int ret = -EINVAL;
3327
3328         target_obj = drm_gem_object_lookup(dev, file_priv,
3329                                            reloc->target_handle);
3330         if (target_obj == NULL)
3331                 return -ENOENT;
3332
3333         target_offset = to_intel_bo(target_obj)->gtt_offset;
3334
3335 #if WATCH_RELOC
3336         DRM_INFO("%s: obj %p offset %08x target %d "
3337                  "read %08x write %08x gtt %08x "
3338                  "presumed %08x delta %08x\n",
3339                  __func__,
3340                  obj,
3341                  (int) reloc->offset,
3342                  (int) reloc->target_handle,
3343                  (int) reloc->read_domains,
3344                  (int) reloc->write_domain,
3345                  (int) target_offset,
3346                  (int) reloc->presumed_offset,
3347                  reloc->delta);
3348 #endif
3349
3350         /* The target buffer should have appeared before us in the
3351          * exec_object list, so it should have a GTT space bound by now.
3352          */
3353         if (target_offset == 0) {
3354                 DRM_ERROR("No GTT space found for object %d\n",
3355                           reloc->target_handle);
3356                 goto err;
3357         }
3358
3359         /* Validate that the target is in a valid r/w GPU domain */
3360         if (reloc->write_domain & (reloc->write_domain - 1)) {
3361                 DRM_ERROR("reloc with multiple write domains: "
3362                           "obj %p target %d offset %d "
3363                           "read %08x write %08x",
3364                           obj, reloc->target_handle,
3365                           (int) reloc->offset,
3366                           reloc->read_domains,
3367                           reloc->write_domain);
3368                 goto err;
3369         }
3370         if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3371             reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3372                 DRM_ERROR("reloc with read/write CPU domains: "
3373                           "obj %p target %d offset %d "
3374                           "read %08x write %08x",
3375                           obj, reloc->target_handle,
3376                           (int) reloc->offset,
3377                           reloc->read_domains,
3378                           reloc->write_domain);
3379                 goto err;
3380         }
3381         if (reloc->write_domain && target_obj->pending_write_domain &&
3382             reloc->write_domain != target_obj->pending_write_domain) {
3383                 DRM_ERROR("Write domain conflict: "
3384                           "obj %p target %d offset %d "
3385                           "new %08x old %08x\n",
3386                           obj, reloc->target_handle,
3387                           (int) reloc->offset,
3388                           reloc->write_domain,
3389                           target_obj->pending_write_domain);
3390                 goto err;
3391         }
3392
3393         target_obj->pending_read_domains |= reloc->read_domains;
3394         target_obj->pending_write_domain |= reloc->write_domain;
3395
3396         /* If the relocation already has the right value in it, no
3397          * more work needs to be done.
3398          */
3399         if (target_offset == reloc->presumed_offset)
3400                 goto out;
3401
3402         /* Check that the relocation address is valid... */
3403         if (reloc->offset > obj->base.size - 4) {
3404                 DRM_ERROR("Relocation beyond object bounds: "
3405                           "obj %p target %d offset %d size %d.\n",
3406                           obj, reloc->target_handle,
3407                           (int) reloc->offset,
3408                           (int) obj->base.size);
3409                 goto err;
3410         }
3411         if (reloc->offset & 3) {
3412                 DRM_ERROR("Relocation not 4-byte aligned: "
3413                           "obj %p target %d offset %d.\n",
3414                           obj, reloc->target_handle,
3415                           (int) reloc->offset);
3416                 goto err;
3417         }
3418
3419         /* and points to somewhere within the target object. */
3420         if (reloc->delta >= target_obj->size) {
3421                 DRM_ERROR("Relocation beyond target object bounds: "
3422                           "obj %p target %d delta %d size %d.\n",
3423                           obj, reloc->target_handle,
3424                           (int) reloc->delta,
3425                           (int) target_obj->size);
3426                 goto err;
3427         }
3428
3429         reloc->delta += target_offset;
3430         if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
3431                 uint32_t page_offset = reloc->offset & ~PAGE_MASK;
3432                 char *vaddr;
3433
3434                 vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]);
3435                 *(uint32_t *)(vaddr + page_offset) = reloc->delta;
3436                 kunmap_atomic(vaddr);
3437         } else {
3438                 struct drm_i915_private *dev_priv = dev->dev_private;
3439                 uint32_t __iomem *reloc_entry;
3440                 void __iomem *reloc_page;
3441
3442                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3443                 if (ret)
3444                         goto err;
3445
3446                 /* Map the page containing the relocation we're going to perform.  */
3447                 reloc->offset += obj->gtt_offset;
3448                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3449                                                       reloc->offset & PAGE_MASK);
3450                 reloc_entry = (uint32_t __iomem *)
3451                         (reloc_page + (reloc->offset & ~PAGE_MASK));
3452                 iowrite32(reloc->delta, reloc_entry);
3453                 io_mapping_unmap_atomic(reloc_page);
3454         }
3455
3456         /* and update the user's relocation entry */
3457         reloc->presumed_offset = target_offset;
3458
3459 out:
3460         ret = 0;
3461 err:
3462         drm_gem_object_unreference(target_obj);
3463         return ret;
3464 }
3465
3466 static int
3467 i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
3468                                     struct drm_file *file_priv,
3469                                     struct drm_i915_gem_exec_object2 *entry)
3470 {
3471         struct drm_i915_gem_relocation_entry __user *user_relocs;
3472         int i, ret;
3473
3474         user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
3475         for (i = 0; i < entry->relocation_count; i++) {
3476                 struct drm_i915_gem_relocation_entry reloc;
3477
3478                 if (__copy_from_user_inatomic(&reloc,
3479                                               user_relocs+i,
3480                                               sizeof(reloc)))
3481                         return -EFAULT;
3482
3483                 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &reloc);
3484                 if (ret)
3485                         return ret;
3486
3487                 if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset,
3488                                             &reloc.presumed_offset,
3489                                             sizeof(reloc.presumed_offset)))
3490                         return -EFAULT;
3491         }
3492
3493         return 0;
3494 }
3495
3496 static int
3497 i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj,
3498                                          struct drm_file *file_priv,
3499                                          struct drm_i915_gem_exec_object2 *entry,
3500                                          struct drm_i915_gem_relocation_entry *relocs)
3501 {
3502         int i, ret;
3503
3504         for (i = 0; i < entry->relocation_count; i++) {
3505                 ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &relocs[i]);
3506                 if (ret)
3507                         return ret;
3508         }
3509
3510         return 0;
3511 }
3512
3513 static int
3514 i915_gem_execbuffer_relocate(struct drm_device *dev,
3515                              struct drm_file *file,
3516                              struct drm_i915_gem_object **object_list,
3517                              struct drm_i915_gem_exec_object2 *exec_list,
3518                              int count)
3519 {
3520         int i, ret;
3521
3522         for (i = 0; i < count; i++) {
3523                 struct drm_i915_gem_object *obj = object_list[i];
3524                 obj->base.pending_read_domains = 0;
3525                 obj->base.pending_write_domain = 0;
3526                 ret = i915_gem_execbuffer_relocate_object(obj, file,
3527                                                           &exec_list[i]);
3528                 if (ret)
3529                         return ret;
3530         }
3531
3532         return 0;
3533 }
3534
3535 static int
3536 i915_gem_execbuffer_reserve(struct drm_device *dev,
3537                             struct drm_file *file,
3538                             struct drm_i915_gem_object **object_list,
3539                             struct drm_i915_gem_exec_object2 *exec_list,
3540                             int count)
3541 {
3542         struct drm_i915_private *dev_priv = dev->dev_private;
3543         int ret, i, retry;
3544
3545         /* attempt to pin all of the buffers into the GTT */
3546         retry = 0;
3547         do {
3548                 ret = 0;
3549                 for (i = 0; i < count; i++) {
3550                         struct drm_i915_gem_exec_object2 *entry = &exec_list[i];
3551                         struct drm_i915_gem_object *obj = object_list[i];
3552                         bool need_fence =
3553                                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3554                                 obj->tiling_mode != I915_TILING_NONE;
3555
3556                         /* g33/pnv can't fence buffers in the unmappable part */
3557                         bool need_mappable =
3558                                 entry->relocation_count ? true : need_fence;
3559
3560                         /* Check fence reg constraints and rebind if necessary */
3561                         if (need_mappable && !obj->map_and_fenceable) {
3562                                 ret = i915_gem_object_unbind(obj);
3563                                 if (ret)
3564                                         break;
3565                         }
3566
3567                         ret = i915_gem_object_pin(obj,
3568                                                   entry->alignment,
3569                                                   need_mappable);
3570                         if (ret)
3571                                 break;
3572
3573                         /*
3574                          * Pre-965 chips need a fence register set up in order
3575                          * to properly handle blits to/from tiled surfaces.
3576                          */
3577                         if (need_fence) {
3578                                 ret = i915_gem_object_get_fence_reg(obj, true);
3579                                 if (ret) {
3580                                         i915_gem_object_unpin(obj);
3581                                         break;
3582                                 }
3583
3584                                 dev_priv->fence_regs[obj->fence_reg].gpu = true;
3585                         }
3586
3587                         entry->offset = obj->gtt_offset;
3588                 }
3589
3590                 while (i--)
3591                         i915_gem_object_unpin(object_list[i]);
3592
3593                 if (ret != -ENOSPC || retry > 1)
3594                         return ret;
3595
3596                 /* First attempt, just clear anything that is purgeable.
3597                  * Second attempt, clear the entire GTT.
3598                  */
3599                 ret = i915_gem_evict_everything(dev, retry == 0);
3600                 if (ret)
3601                         return ret;
3602
3603                 retry++;
3604         } while (1);
3605 }
3606
3607 static int
3608 i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
3609                                   struct drm_file *file,
3610                                   struct drm_i915_gem_object **object_list,
3611                                   struct drm_i915_gem_exec_object2 *exec_list,
3612                                   int count)
3613 {
3614         struct drm_i915_gem_relocation_entry *reloc;
3615         int i, total, ret;
3616
3617         for (i = 0; i < count; i++)
3618                 object_list[i]->in_execbuffer = false;
3619
3620         mutex_unlock(&dev->struct_mutex);
3621
3622         total = 0;
3623         for (i = 0; i < count; i++)
3624                 total += exec_list[i].relocation_count;
3625
3626         reloc = drm_malloc_ab(total, sizeof(*reloc));
3627         if (reloc == NULL) {
3628                 mutex_lock(&dev->struct_mutex);
3629                 return -ENOMEM;
3630         }
3631
3632         total = 0;
3633         for (i = 0; i < count; i++) {
3634                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3635
3636                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3637
3638                 if (copy_from_user(reloc+total, user_relocs,
3639                                    exec_list[i].relocation_count *
3640                                    sizeof(*reloc))) {
3641                         ret = -EFAULT;
3642                         mutex_lock(&dev->struct_mutex);
3643                         goto err;
3644                 }
3645
3646                 total += exec_list[i].relocation_count;
3647         }
3648
3649         ret = i915_mutex_lock_interruptible(dev);
3650         if (ret) {
3651                 mutex_lock(&dev->struct_mutex);
3652                 goto err;
3653         }
3654
3655         ret = i915_gem_execbuffer_reserve(dev, file,
3656                                           object_list, exec_list,
3657                                           count);
3658         if (ret)
3659                 goto err;
3660
3661         total = 0;
3662         for (i = 0; i < count; i++) {
3663                 struct drm_i915_gem_object *obj = object_list[i];
3664                 obj->base.pending_read_domains = 0;
3665                 obj->base.pending_write_domain = 0;
3666                 ret = i915_gem_execbuffer_relocate_object_slow(obj, file,
3667                                                                &exec_list[i],
3668                                                                reloc + total);
3669                 if (ret)
3670                         goto err;
3671
3672                 total += exec_list[i].relocation_count;
3673         }
3674
3675         /* Leave the user relocations as are, this is the painfully slow path,
3676          * and we want to avoid the complication of dropping the lock whilst
3677          * having buffers reserved in the aperture and so causing spurious
3678          * ENOSPC for random operations.
3679          */
3680
3681 err:
3682         drm_free_large(reloc);
3683         return ret;
3684 }
3685
3686 static int
3687 i915_gem_execbuffer_move_to_gpu(struct drm_device *dev,
3688                                 struct drm_file *file,
3689                                 struct intel_ring_buffer *ring,
3690                                 struct drm_i915_gem_object **objects,
3691                                 int count)
3692 {
3693         struct change_domains cd;
3694         int ret, i;
3695
3696         cd.invalidate_domains = 0;
3697         cd.flush_domains = 0;
3698         cd.flush_rings = 0;
3699         for (i = 0; i < count; i++)
3700                 i915_gem_object_set_to_gpu_domain(objects[i], ring, &cd);
3701
3702         if (cd.invalidate_domains | cd.flush_domains) {
3703 #if WATCH_EXEC
3704                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3705                           __func__,
3706                          cd.invalidate_domains,
3707                          cd.flush_domains);
3708 #endif
3709                 i915_gem_flush(dev,
3710                                cd.invalidate_domains,
3711                                cd.flush_domains,
3712                                cd.flush_rings);
3713         }
3714
3715         for (i = 0; i < count; i++) {
3716                 struct drm_i915_gem_object *obj = objects[i];
3717                 /* XXX replace with semaphores */
3718                 if (obj->ring && ring != obj->ring) {
3719                         ret = i915_gem_object_wait_rendering(obj, true);
3720                         if (ret)
3721                                 return ret;
3722                 }
3723         }
3724
3725         return 0;
3726 }
3727
3728 /* Throttle our rendering by waiting until the ring has completed our requests
3729  * emitted over 20 msec ago.
3730  *
3731  * Note that if we were to use the current jiffies each time around the loop,
3732  * we wouldn't escape the function with any frames outstanding if the time to
3733  * render a frame was over 20ms.
3734  *
3735  * This should get us reasonable parallelism between CPU and GPU but also
3736  * relatively low latency when blocking on a particular request to finish.
3737  */
3738 static int
3739 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3740 {
3741         struct drm_i915_private *dev_priv = dev->dev_private;
3742         struct drm_i915_file_private *file_priv = file->driver_priv;
3743         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3744         struct drm_i915_gem_request *request;
3745         struct intel_ring_buffer *ring = NULL;
3746         u32 seqno = 0;
3747         int ret;
3748
3749         spin_lock(&file_priv->mm.lock);
3750         list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3751                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3752                         break;
3753
3754                 ring = request->ring;
3755                 seqno = request->seqno;
3756         }
3757         spin_unlock(&file_priv->mm.lock);
3758
3759         if (seqno == 0)
3760                 return 0;
3761
3762         ret = 0;
3763         if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3764                 /* And wait for the seqno passing without holding any locks and
3765                  * causing extra latency for others. This is safe as the irq
3766                  * generation is designed to be run atomically and so is
3767                  * lockless.
3768                  */
3769                 ring->user_irq_get(ring);
3770                 ret = wait_event_interruptible(ring->irq_queue,
3771                                                i915_seqno_passed(ring->get_seqno(ring), seqno)
3772                                                || atomic_read(&dev_priv->mm.wedged));
3773                 ring->user_irq_put(ring);
3774
3775                 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3776                         ret = -EIO;
3777         }
3778
3779         if (ret == 0)
3780                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3781
3782         return ret;
3783 }
3784
3785 static int
3786 i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3787                           uint64_t exec_offset)
3788 {
3789         uint32_t exec_start, exec_len;
3790
3791         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3792         exec_len = (uint32_t) exec->batch_len;
3793
3794         if ((exec_start | exec_len) & 0x7)
3795                 return -EINVAL;
3796
3797         if (!exec_start)
3798                 return -EINVAL;
3799
3800         return 0;
3801 }
3802
3803 static int
3804 validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3805                    int count)
3806 {
3807         int i;
3808
3809         for (i = 0; i < count; i++) {
3810                 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3811                 int length; /* limited by fault_in_pages_readable() */
3812
3813                 /* First check for malicious input causing overflow */
3814                 if (exec[i].relocation_count >
3815                     INT_MAX / sizeof(struct drm_i915_gem_relocation_entry))
3816                         return -EINVAL;
3817
3818                 length = exec[i].relocation_count *
3819                         sizeof(struct drm_i915_gem_relocation_entry);
3820                 if (!access_ok(VERIFY_READ, ptr, length))
3821                         return -EFAULT;
3822
3823                 /* we may also need to update the presumed offsets */
3824                 if (!access_ok(VERIFY_WRITE, ptr, length))
3825                         return -EFAULT;
3826
3827                 if (fault_in_pages_readable(ptr, length))
3828                         return -EFAULT;
3829         }
3830
3831         return 0;
3832 }
3833
3834 static int
3835 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3836                        struct drm_file *file,
3837                        struct drm_i915_gem_execbuffer2 *args,
3838                        struct drm_i915_gem_exec_object2 *exec_list)
3839 {
3840         drm_i915_private_t *dev_priv = dev->dev_private;
3841         struct drm_i915_gem_object **object_list = NULL;
3842         struct drm_i915_gem_object *batch_obj;
3843         struct drm_clip_rect *cliprects = NULL;
3844         struct drm_i915_gem_request *request = NULL;
3845         int ret, i, flips;
3846         uint64_t exec_offset;
3847
3848         struct intel_ring_buffer *ring = NULL;
3849
3850         ret = i915_gem_check_is_wedged(dev);
3851         if (ret)
3852                 return ret;
3853
3854         ret = validate_exec_list(exec_list, args->buffer_count);
3855         if (ret)
3856                 return ret;
3857
3858 #if WATCH_EXEC
3859         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3860                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3861 #endif
3862         switch (args->flags & I915_EXEC_RING_MASK) {
3863         case I915_EXEC_DEFAULT:
3864         case I915_EXEC_RENDER:
3865                 ring = &dev_priv->render_ring;
3866                 break;
3867         case I915_EXEC_BSD:
3868                 if (!HAS_BSD(dev)) {
3869                         DRM_ERROR("execbuf with invalid ring (BSD)\n");
3870                         return -EINVAL;
3871                 }
3872                 ring = &dev_priv->bsd_ring;
3873                 break;
3874         case I915_EXEC_BLT:
3875                 if (!HAS_BLT(dev)) {
3876                         DRM_ERROR("execbuf with invalid ring (BLT)\n");
3877                         return -EINVAL;
3878                 }
3879                 ring = &dev_priv->blt_ring;
3880                 break;
3881         default:
3882                 DRM_ERROR("execbuf with unknown ring: %d\n",
3883                           (int)(args->flags & I915_EXEC_RING_MASK));
3884                 return -EINVAL;
3885         }
3886
3887         if (args->buffer_count < 1) {
3888                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3889                 return -EINVAL;
3890         }
3891         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3892         if (object_list == NULL) {
3893                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3894                           args->buffer_count);
3895                 ret = -ENOMEM;
3896                 goto pre_mutex_err;
3897         }
3898
3899         if (args->num_cliprects != 0) {
3900                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3901                                     GFP_KERNEL);
3902                 if (cliprects == NULL) {
3903                         ret = -ENOMEM;
3904                         goto pre_mutex_err;
3905                 }
3906
3907                 ret = copy_from_user(cliprects,
3908                                      (struct drm_clip_rect __user *)
3909                                      (uintptr_t) args->cliprects_ptr,
3910                                      sizeof(*cliprects) * args->num_cliprects);
3911                 if (ret != 0) {
3912                         DRM_ERROR("copy %d cliprects failed: %d\n",
3913                                   args->num_cliprects, ret);
3914                         ret = -EFAULT;
3915                         goto pre_mutex_err;
3916                 }
3917         }
3918
3919         request = kzalloc(sizeof(*request), GFP_KERNEL);
3920         if (request == NULL) {
3921                 ret = -ENOMEM;
3922                 goto pre_mutex_err;
3923         }
3924
3925         ret = i915_mutex_lock_interruptible(dev);
3926         if (ret)
3927                 goto pre_mutex_err;
3928
3929         if (dev_priv->mm.suspended) {
3930                 mutex_unlock(&dev->struct_mutex);
3931                 ret = -EBUSY;
3932                 goto pre_mutex_err;
3933         }
3934
3935         /* Look up object handles */
3936         for (i = 0; i < args->buffer_count; i++) {
3937                 struct drm_i915_gem_object *obj;
3938
3939                 obj = to_intel_bo (drm_gem_object_lookup(dev, file,
3940                                                          exec_list[i].handle));
3941                 if (obj == NULL) {
3942                         DRM_ERROR("Invalid object handle %d at index %d\n",
3943                                    exec_list[i].handle, i);
3944                         /* prevent error path from reading uninitialized data */
3945                         args->buffer_count = i;
3946                         ret = -ENOENT;
3947                         goto err;
3948                 }
3949                 object_list[i] = obj;
3950
3951                 if (obj->in_execbuffer) {
3952                         DRM_ERROR("Object %p appears more than once in object list\n",
3953                                    obj);
3954                         /* prevent error path from reading uninitialized data */
3955                         args->buffer_count = i + 1;
3956                         ret = -EINVAL;
3957                         goto err;
3958                 }
3959                 obj->in_execbuffer = true;
3960         }
3961
3962         /* Move the objects en-masse into the GTT, evicting if necessary. */
3963         ret = i915_gem_execbuffer_reserve(dev, file,
3964                                           object_list, exec_list,
3965                                           args->buffer_count);
3966         if (ret)
3967                 goto err;
3968
3969         /* The objects are in their final locations, apply the relocations. */
3970         ret = i915_gem_execbuffer_relocate(dev, file,
3971                                            object_list, exec_list,
3972                                            args->buffer_count);
3973         if (ret) {
3974                 if (ret == -EFAULT) {
3975                         ret = i915_gem_execbuffer_relocate_slow(dev, file,
3976                                                                 object_list,
3977                                                                 exec_list,
3978                                                                 args->buffer_count);
3979                         BUG_ON(!mutex_is_locked(&dev->struct_mutex));
3980                 }
3981                 if (ret)
3982                         goto err;
3983         }
3984
3985         /* Set the pending read domains for the batch buffer to COMMAND */
3986         batch_obj = object_list[args->buffer_count-1];
3987         if (batch_obj->base.pending_write_domain) {
3988                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3989                 ret = -EINVAL;
3990                 goto err;
3991         }
3992         batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3993
3994         /* Sanity check the batch buffer */
3995         exec_offset = batch_obj->gtt_offset;
3996         ret = i915_gem_check_execbuffer(args, exec_offset);
3997         if (ret != 0) {
3998                 DRM_ERROR("execbuf with invalid offset/length\n");
3999                 goto err;
4000         }
4001
4002         ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring,
4003                                               object_list, args->buffer_count);
4004         if (ret)
4005                 goto err;
4006
4007 #if WATCH_COHERENCY
4008         for (i = 0; i < args->buffer_count; i++) {
4009                 i915_gem_object_check_coherency(object_list[i],
4010                                                 exec_list[i].handle);
4011         }
4012 #endif
4013
4014 #if WATCH_EXEC
4015         i915_gem_dump_object(batch_obj,
4016                               args->batch_len,
4017                               __func__,
4018                               ~0);
4019 #endif
4020
4021         /* Check for any pending flips. As we only maintain a flip queue depth
4022          * of 1, we can simply insert a WAIT for the next display flip prior
4023          * to executing the batch and avoid stalling the CPU.
4024          */
4025         flips = 0;
4026         for (i = 0; i < args->buffer_count; i++) {
4027                 if (object_list[i]->base.write_domain)
4028                         flips |= atomic_read(&object_list[i]->pending_flip);
4029         }
4030         if (flips) {
4031                 int plane, flip_mask;
4032
4033                 for (plane = 0; flips >> plane; plane++) {
4034                         if (((flips >> plane) & 1) == 0)
4035                                 continue;
4036
4037                         if (plane)
4038                                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
4039                         else
4040                                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
4041
4042                         ret = intel_ring_begin(ring, 2);
4043                         if (ret)
4044                                 goto err;
4045
4046                         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
4047                         intel_ring_emit(ring, MI_NOOP);
4048                         intel_ring_advance(ring);
4049                 }
4050         }
4051
4052         /* Exec the batchbuffer */
4053         ret = ring->dispatch_execbuffer(ring, args, cliprects, exec_offset);
4054         if (ret) {
4055                 DRM_ERROR("dispatch failed %d\n", ret);
4056                 goto err;
4057         }
4058
4059         for (i = 0; i < args->buffer_count; i++) {
4060                 struct drm_i915_gem_object *obj = object_list[i];
4061
4062                 obj->base.read_domains = obj->base.pending_read_domains;
4063                 obj->base.write_domain = obj->base.pending_write_domain;
4064
4065                 i915_gem_object_move_to_active(obj, ring);
4066                 if (obj->base.write_domain) {
4067                         obj->dirty = 1;
4068                         list_move_tail(&obj->gpu_write_list,
4069                                        &ring->gpu_write_list);
4070                         intel_mark_busy(dev, obj);
4071                 }
4072
4073                 trace_i915_gem_object_change_domain(obj,
4074                                                     obj->base.read_domains,
4075                                                     obj->base.write_domain);
4076         }
4077
4078         /*
4079          * Ensure that the commands in the batch buffer are
4080          * finished before the interrupt fires
4081          */
4082         i915_retire_commands(dev, ring);
4083
4084         if (i915_add_request(dev, file, request, ring))
4085                 i915_gem_next_request_seqno(dev, ring);
4086         else
4087                 request = NULL;
4088
4089 err:
4090         for (i = 0; i < args->buffer_count; i++) {
4091                 object_list[i]->in_execbuffer = false;
4092                 drm_gem_object_unreference(&object_list[i]->base);
4093         }
4094
4095         mutex_unlock(&dev->struct_mutex);
4096
4097 pre_mutex_err:
4098         drm_free_large(object_list);
4099         kfree(cliprects);
4100         kfree(request);
4101
4102         return ret;
4103 }
4104
4105 /*
4106  * Legacy execbuffer just creates an exec2 list from the original exec object
4107  * list array and passes it to the real function.
4108  */
4109 int
4110 i915_gem_execbuffer(struct drm_device *dev, void *data,
4111                     struct drm_file *file)
4112 {
4113         struct drm_i915_gem_execbuffer *args = data;
4114         struct drm_i915_gem_execbuffer2 exec2;
4115         struct drm_i915_gem_exec_object *exec_list = NULL;
4116         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4117         int ret, i;
4118
4119 #if WATCH_EXEC
4120         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4121                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4122 #endif
4123
4124         if (args->buffer_count < 1) {
4125                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4126                 return -EINVAL;
4127         }
4128
4129         /* Copy in the exec list from userland */
4130         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4131         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4132         if (exec_list == NULL || exec2_list == NULL) {
4133                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4134                           args->buffer_count);
4135                 drm_free_large(exec_list);
4136                 drm_free_large(exec2_list);
4137                 return -ENOMEM;
4138         }
4139         ret = copy_from_user(exec_list,
4140                              (struct drm_i915_relocation_entry __user *)
4141                              (uintptr_t) args->buffers_ptr,
4142                              sizeof(*exec_list) * args->buffer_count);
4143         if (ret != 0) {
4144                 DRM_ERROR("copy %d exec entries failed %d\n",
4145                           args->buffer_count, ret);
4146                 drm_free_large(exec_list);
4147                 drm_free_large(exec2_list);
4148                 return -EFAULT;
4149         }
4150
4151         for (i = 0; i < args->buffer_count; i++) {
4152                 exec2_list[i].handle = exec_list[i].handle;
4153                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4154                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4155                 exec2_list[i].alignment = exec_list[i].alignment;
4156                 exec2_list[i].offset = exec_list[i].offset;
4157                 if (INTEL_INFO(dev)->gen < 4)
4158                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4159                 else
4160                         exec2_list[i].flags = 0;
4161         }
4162
4163         exec2.buffers_ptr = args->buffers_ptr;
4164         exec2.buffer_count = args->buffer_count;
4165         exec2.batch_start_offset = args->batch_start_offset;
4166         exec2.batch_len = args->batch_len;
4167         exec2.DR1 = args->DR1;
4168         exec2.DR4 = args->DR4;
4169         exec2.num_cliprects = args->num_cliprects;
4170         exec2.cliprects_ptr = args->cliprects_ptr;
4171         exec2.flags = I915_EXEC_RENDER;
4172
4173         ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
4174         if (!ret) {
4175                 /* Copy the new buffer offsets back to the user's exec list. */
4176                 for (i = 0; i < args->buffer_count; i++)
4177                         exec_list[i].offset = exec2_list[i].offset;
4178                 /* ... and back out to userspace */
4179                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4180                                    (uintptr_t) args->buffers_ptr,
4181                                    exec_list,
4182                                    sizeof(*exec_list) * args->buffer_count);
4183                 if (ret) {
4184                         ret = -EFAULT;
4185                         DRM_ERROR("failed to copy %d exec entries "
4186                                   "back to user (%d)\n",
4187                                   args->buffer_count, ret);
4188                 }
4189         }
4190
4191         drm_free_large(exec_list);
4192         drm_free_large(exec2_list);
4193         return ret;
4194 }
4195
4196 int
4197 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4198                      struct drm_file *file)
4199 {
4200         struct drm_i915_gem_execbuffer2 *args = data;
4201         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4202         int ret;
4203
4204 #if WATCH_EXEC
4205         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4206                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4207 #endif
4208
4209         if (args->buffer_count < 1) {
4210                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4211                 return -EINVAL;
4212         }
4213
4214         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4215         if (exec2_list == NULL) {
4216                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4217                           args->buffer_count);
4218                 return -ENOMEM;
4219         }
4220         ret = copy_from_user(exec2_list,
4221                              (struct drm_i915_relocation_entry __user *)
4222                              (uintptr_t) args->buffers_ptr,
4223                              sizeof(*exec2_list) * args->buffer_count);
4224         if (ret != 0) {
4225                 DRM_ERROR("copy %d exec entries failed %d\n",
4226                           args->buffer_count, ret);
4227                 drm_free_large(exec2_list);
4228                 return -EFAULT;
4229         }
4230
4231         ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
4232         if (!ret) {
4233                 /* Copy the new buffer offsets back to the user's exec list. */
4234                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4235                                    (uintptr_t) args->buffers_ptr,
4236                                    exec2_list,
4237                                    sizeof(*exec2_list) * args->buffer_count);
4238                 if (ret) {
4239                         ret = -EFAULT;
4240                         DRM_ERROR("failed to copy %d exec entries "
4241                                   "back to user (%d)\n",
4242                                   args->buffer_count, ret);
4243                 }
4244         }
4245
4246         drm_free_large(exec2_list);
4247         return ret;
4248 }
4249
4250 int
4251 i915_gem_object_pin(struct drm_i915_gem_object *obj,
4252                     uint32_t alignment,
4253                     bool map_and_fenceable)
4254 {
4255         struct drm_device *dev = obj->base.dev;
4256         struct drm_i915_private *dev_priv = dev->dev_private;
4257         int ret;
4258
4259         BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4260         BUG_ON(map_and_fenceable && !map_and_fenceable);
4261         WARN_ON(i915_verify_lists(dev));
4262
4263         if (obj->gtt_space != NULL) {
4264                 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
4265                     (map_and_fenceable && !obj->map_and_fenceable)) {
4266                         WARN(obj->pin_count,
4267                              "bo is already pinned with incorrect alignment:"
4268                              " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
4269                              " obj->map_and_fenceable=%d\n",
4270                              obj->gtt_offset, alignment,
4271                              map_and_fenceable,
4272                              obj->map_and_fenceable);
4273                         ret = i915_gem_object_unbind(obj);
4274                         if (ret)
4275                                 return ret;
4276                 }
4277         }
4278
4279         if (obj->gtt_space == NULL) {
4280                 ret = i915_gem_object_bind_to_gtt(obj, alignment,
4281                                                   map_and_fenceable);
4282                 if (ret)
4283                         return ret;
4284         }
4285
4286         if (obj->pin_count++ == 0) {
4287                 i915_gem_info_add_pin(dev_priv, obj, map_and_fenceable);
4288                 if (!obj->active)
4289                         list_move_tail(&obj->mm_list,
4290                                        &dev_priv->mm.pinned_list);
4291         }
4292         BUG_ON(!obj->pin_mappable && map_and_fenceable);
4293
4294         WARN_ON(i915_verify_lists(dev));
4295         return 0;
4296 }
4297
4298 void
4299 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
4300 {
4301         struct drm_device *dev = obj->base.dev;
4302         drm_i915_private_t *dev_priv = dev->dev_private;
4303
4304         WARN_ON(i915_verify_lists(dev));
4305         BUG_ON(obj->pin_count == 0);
4306         BUG_ON(obj->gtt_space == NULL);
4307
4308         if (--obj->pin_count == 0) {
4309                 if (!obj->active)
4310                         list_move_tail(&obj->mm_list,
4311                                        &dev_priv->mm.inactive_list);
4312                 i915_gem_info_remove_pin(dev_priv, obj);
4313         }
4314         WARN_ON(i915_verify_lists(dev));
4315 }
4316
4317 int
4318 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4319                    struct drm_file *file)
4320 {
4321         struct drm_i915_gem_pin *args = data;
4322         struct drm_i915_gem_object *obj;
4323         int ret;
4324
4325         ret = i915_mutex_lock_interruptible(dev);
4326         if (ret)
4327                 return ret;
4328
4329         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4330         if (obj == NULL) {
4331                 ret = -ENOENT;
4332                 goto unlock;
4333         }
4334
4335         if (obj->madv != I915_MADV_WILLNEED) {
4336                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4337                 ret = -EINVAL;
4338                 goto out;
4339         }
4340
4341         if (obj->pin_filp != NULL && obj->pin_filp != file) {
4342                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4343                           args->handle);
4344                 ret = -EINVAL;
4345                 goto out;
4346         }
4347
4348         obj->user_pin_count++;
4349         obj->pin_filp = file;
4350         if (obj->user_pin_count == 1) {
4351                 ret = i915_gem_object_pin(obj, args->alignment, true);
4352                 if (ret)
4353                         goto out;
4354         }
4355
4356         /* XXX - flush the CPU caches for pinned objects
4357          * as the X server doesn't manage domains yet
4358          */
4359         i915_gem_object_flush_cpu_write_domain(obj);
4360         args->offset = obj->gtt_offset;
4361 out:
4362         drm_gem_object_unreference(&obj->base);
4363 unlock:
4364         mutex_unlock(&dev->struct_mutex);
4365         return ret;
4366 }
4367
4368 int
4369 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4370                      struct drm_file *file)
4371 {
4372         struct drm_i915_gem_pin *args = data;
4373         struct drm_i915_gem_object *obj;
4374         int ret;
4375
4376         ret = i915_mutex_lock_interruptible(dev);
4377         if (ret)
4378                 return ret;
4379
4380         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4381         if (obj == NULL) {
4382                 ret = -ENOENT;
4383                 goto unlock;
4384         }
4385
4386         if (obj->pin_filp != file) {
4387                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4388                           args->handle);
4389                 ret = -EINVAL;
4390                 goto out;
4391         }
4392         obj->user_pin_count--;
4393         if (obj->user_pin_count == 0) {
4394                 obj->pin_filp = NULL;
4395                 i915_gem_object_unpin(obj);
4396         }
4397
4398 out:
4399         drm_gem_object_unreference(&obj->base);
4400 unlock:
4401         mutex_unlock(&dev->struct_mutex);
4402         return ret;
4403 }
4404
4405 int
4406 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4407                     struct drm_file *file)
4408 {
4409         struct drm_i915_gem_busy *args = data;
4410         struct drm_i915_gem_object *obj;
4411         int ret;
4412
4413         ret = i915_mutex_lock_interruptible(dev);
4414         if (ret)
4415                 return ret;
4416
4417         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4418         if (obj == NULL) {
4419                 ret = -ENOENT;
4420                 goto unlock;
4421         }
4422
4423         /* Count all active objects as busy, even if they are currently not used
4424          * by the gpu. Users of this interface expect objects to eventually
4425          * become non-busy without any further actions, therefore emit any
4426          * necessary flushes here.
4427          */
4428         args->busy = obj->active;
4429         if (args->busy) {
4430                 /* Unconditionally flush objects, even when the gpu still uses this
4431                  * object. Userspace calling this function indicates that it wants to
4432                  * use this buffer rather sooner than later, so issuing the required
4433                  * flush earlier is beneficial.
4434                  */
4435                 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
4436                         i915_gem_flush_ring(dev, obj->ring,
4437                                             0, obj->base.write_domain);
4438
4439                 /* Update the active list for the hardware's current position.
4440                  * Otherwise this only updates on a delayed timer or when irqs
4441                  * are actually unmasked, and our working set ends up being
4442                  * larger than required.
4443                  */
4444                 i915_gem_retire_requests_ring(dev, obj->ring);
4445
4446                 args->busy = obj->active;
4447         }
4448
4449         drm_gem_object_unreference(&obj->base);
4450 unlock:
4451         mutex_unlock(&dev->struct_mutex);
4452         return ret;
4453 }
4454
4455 int
4456 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4457                         struct drm_file *file_priv)
4458 {
4459     return i915_gem_ring_throttle(dev, file_priv);
4460 }
4461
4462 int
4463 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4464                        struct drm_file *file_priv)
4465 {
4466         struct drm_i915_gem_madvise *args = data;
4467         struct drm_i915_gem_object *obj;
4468         int ret;
4469
4470         switch (args->madv) {
4471         case I915_MADV_DONTNEED:
4472         case I915_MADV_WILLNEED:
4473             break;
4474         default:
4475             return -EINVAL;
4476         }
4477
4478         ret = i915_mutex_lock_interruptible(dev);
4479         if (ret)
4480                 return ret;
4481
4482         obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4483         if (obj == NULL) {
4484                 ret = -ENOENT;
4485                 goto unlock;
4486         }
4487
4488         if (obj->pin_count) {
4489                 ret = -EINVAL;
4490                 goto out;
4491         }
4492
4493         if (obj->madv != __I915_MADV_PURGED)
4494                 obj->madv = args->madv;
4495
4496         /* if the object is no longer bound, discard its backing storage */
4497         if (i915_gem_object_is_purgeable(obj) &&
4498             obj->gtt_space == NULL)
4499                 i915_gem_object_truncate(obj);
4500
4501         args->retained = obj->madv != __I915_MADV_PURGED;
4502
4503 out:
4504         drm_gem_object_unreference(&obj->base);
4505 unlock:
4506         mutex_unlock(&dev->struct_mutex);
4507         return ret;
4508 }
4509
4510 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4511                                                   size_t size)
4512 {
4513         struct drm_i915_private *dev_priv = dev->dev_private;
4514         struct drm_i915_gem_object *obj;
4515
4516         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4517         if (obj == NULL)
4518                 return NULL;
4519
4520         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4521                 kfree(obj);
4522                 return NULL;
4523         }
4524
4525         i915_gem_info_add_obj(dev_priv, size);
4526
4527         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4528         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4529
4530         obj->agp_type = AGP_USER_MEMORY;
4531         obj->base.driver_private = NULL;
4532         obj->fence_reg = I915_FENCE_REG_NONE;
4533         INIT_LIST_HEAD(&obj->mm_list);
4534         INIT_LIST_HEAD(&obj->gtt_list);
4535         INIT_LIST_HEAD(&obj->ring_list);
4536         INIT_LIST_HEAD(&obj->gpu_write_list);
4537         obj->madv = I915_MADV_WILLNEED;
4538         /* Avoid an unnecessary call to unbind on the first bind. */
4539         obj->map_and_fenceable = true;
4540
4541         return obj;
4542 }
4543
4544 int i915_gem_init_object(struct drm_gem_object *obj)
4545 {
4546         BUG();
4547
4548         return 0;
4549 }
4550
4551 static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
4552 {
4553         struct drm_device *dev = obj->base.dev;
4554         drm_i915_private_t *dev_priv = dev->dev_private;
4555         int ret;
4556
4557         ret = i915_gem_object_unbind(obj);
4558         if (ret == -ERESTARTSYS) {
4559                 list_move(&obj->mm_list,
4560                           &dev_priv->mm.deferred_free_list);
4561                 return;
4562         }
4563
4564         if (obj->base.map_list.map)
4565                 i915_gem_free_mmap_offset(obj);
4566
4567         drm_gem_object_release(&obj->base);
4568         i915_gem_info_remove_obj(dev_priv, obj->base.size);
4569
4570         kfree(obj->page_cpu_valid);
4571         kfree(obj->bit_17);
4572         kfree(obj);
4573 }
4574
4575 void i915_gem_free_object(struct drm_gem_object *gem_obj)
4576 {
4577         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4578         struct drm_device *dev = obj->base.dev;
4579
4580         trace_i915_gem_object_destroy(obj);
4581
4582         while (obj->pin_count > 0)
4583                 i915_gem_object_unpin(obj);
4584
4585         if (obj->phys_obj)
4586                 i915_gem_detach_phys_object(dev, obj);
4587
4588         i915_gem_free_object_tail(obj);
4589 }
4590
4591 int
4592 i915_gem_idle(struct drm_device *dev)
4593 {
4594         drm_i915_private_t *dev_priv = dev->dev_private;
4595         int ret;
4596
4597         mutex_lock(&dev->struct_mutex);
4598
4599         if (dev_priv->mm.suspended) {
4600                 mutex_unlock(&dev->struct_mutex);
4601                 return 0;
4602         }
4603
4604         ret = i915_gpu_idle(dev);
4605         if (ret) {
4606                 mutex_unlock(&dev->struct_mutex);
4607                 return ret;
4608         }
4609
4610         /* Under UMS, be paranoid and evict. */
4611         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4612                 ret = i915_gem_evict_inactive(dev, false);
4613                 if (ret) {
4614                         mutex_unlock(&dev->struct_mutex);
4615                         return ret;
4616                 }
4617         }
4618
4619         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4620          * We need to replace this with a semaphore, or something.
4621          * And not confound mm.suspended!
4622          */
4623         dev_priv->mm.suspended = 1;
4624         del_timer_sync(&dev_priv->hangcheck_timer);
4625
4626         i915_kernel_lost_context(dev);
4627         i915_gem_cleanup_ringbuffer(dev);
4628
4629         mutex_unlock(&dev->struct_mutex);
4630
4631         /* Cancel the retire work handler, which should be idle now. */
4632         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4633
4634         return 0;
4635 }
4636
4637 /*
4638  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4639  * over cache flushing.
4640  */
4641 static int
4642 i915_gem_init_pipe_control(struct drm_device *dev)
4643 {
4644         drm_i915_private_t *dev_priv = dev->dev_private;
4645         struct drm_i915_gem_object *obj;
4646         int ret;
4647
4648         obj = i915_gem_alloc_object(dev, 4096);
4649         if (obj == NULL) {
4650                 DRM_ERROR("Failed to allocate seqno page\n");
4651                 ret = -ENOMEM;
4652                 goto err;
4653         }
4654         obj->agp_type = AGP_USER_CACHED_MEMORY;
4655
4656         ret = i915_gem_object_pin(obj, 4096, true);
4657         if (ret)
4658                 goto err_unref;
4659
4660         dev_priv->seqno_gfx_addr = obj->gtt_offset;
4661         dev_priv->seqno_page =  kmap(obj->pages[0]);
4662         if (dev_priv->seqno_page == NULL)
4663                 goto err_unpin;
4664
4665         dev_priv->seqno_obj = obj;
4666         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4667
4668         return 0;
4669
4670 err_unpin:
4671         i915_gem_object_unpin(obj);
4672 err_unref:
4673         drm_gem_object_unreference(&obj->base);
4674 err:
4675         return ret;
4676 }
4677
4678
4679 static void
4680 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4681 {
4682         drm_i915_private_t *dev_priv = dev->dev_private;
4683         struct drm_i915_gem_object *obj;
4684
4685         obj = dev_priv->seqno_obj;
4686         kunmap(obj->pages[0]);
4687         i915_gem_object_unpin(obj);
4688         drm_gem_object_unreference(&obj->base);
4689         dev_priv->seqno_obj = NULL;
4690
4691         dev_priv->seqno_page = NULL;
4692 }
4693
4694 int
4695 i915_gem_init_ringbuffer(struct drm_device *dev)
4696 {
4697         drm_i915_private_t *dev_priv = dev->dev_private;
4698         int ret;
4699
4700         if (HAS_PIPE_CONTROL(dev)) {
4701                 ret = i915_gem_init_pipe_control(dev);
4702                 if (ret)
4703                         return ret;
4704         }
4705
4706         ret = intel_init_render_ring_buffer(dev);
4707         if (ret)
4708                 goto cleanup_pipe_control;
4709
4710         if (HAS_BSD(dev)) {
4711                 ret = intel_init_bsd_ring_buffer(dev);
4712                 if (ret)
4713                         goto cleanup_render_ring;
4714         }
4715
4716         if (HAS_BLT(dev)) {
4717                 ret = intel_init_blt_ring_buffer(dev);
4718                 if (ret)
4719                         goto cleanup_bsd_ring;
4720         }
4721
4722         dev_priv->next_seqno = 1;
4723
4724         return 0;
4725
4726 cleanup_bsd_ring:
4727         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4728 cleanup_render_ring:
4729         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4730 cleanup_pipe_control:
4731         if (HAS_PIPE_CONTROL(dev))
4732                 i915_gem_cleanup_pipe_control(dev);
4733         return ret;
4734 }
4735
4736 void
4737 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4738 {
4739         drm_i915_private_t *dev_priv = dev->dev_private;
4740
4741         intel_cleanup_ring_buffer(&dev_priv->render_ring);
4742         intel_cleanup_ring_buffer(&dev_priv->bsd_ring);
4743         intel_cleanup_ring_buffer(&dev_priv->blt_ring);
4744         if (HAS_PIPE_CONTROL(dev))
4745                 i915_gem_cleanup_pipe_control(dev);
4746 }
4747
4748 int
4749 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4750                        struct drm_file *file_priv)
4751 {
4752         drm_i915_private_t *dev_priv = dev->dev_private;
4753         int ret;
4754
4755         if (drm_core_check_feature(dev, DRIVER_MODESET))
4756                 return 0;
4757
4758         if (atomic_read(&dev_priv->mm.wedged)) {
4759                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4760                 atomic_set(&dev_priv->mm.wedged, 0);
4761         }
4762
4763         mutex_lock(&dev->struct_mutex);
4764         dev_priv->mm.suspended = 0;
4765
4766         ret = i915_gem_init_ringbuffer(dev);
4767         if (ret != 0) {
4768                 mutex_unlock(&dev->struct_mutex);
4769                 return ret;
4770         }
4771
4772         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4773         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4774         BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list));
4775         BUG_ON(!list_empty(&dev_priv->blt_ring.active_list));
4776         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4777         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4778         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4779         BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list));
4780         BUG_ON(!list_empty(&dev_priv->blt_ring.request_list));
4781         mutex_unlock(&dev->struct_mutex);
4782
4783         ret = drm_irq_install(dev);
4784         if (ret)
4785                 goto cleanup_ringbuffer;
4786
4787         return 0;
4788
4789 cleanup_ringbuffer:
4790         mutex_lock(&dev->struct_mutex);
4791         i915_gem_cleanup_ringbuffer(dev);
4792         dev_priv->mm.suspended = 1;
4793         mutex_unlock(&dev->struct_mutex);
4794
4795         return ret;
4796 }
4797
4798 int
4799 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4800                        struct drm_file *file_priv)
4801 {
4802         if (drm_core_check_feature(dev, DRIVER_MODESET))
4803                 return 0;
4804
4805         drm_irq_uninstall(dev);
4806         return i915_gem_idle(dev);
4807 }
4808
4809 void
4810 i915_gem_lastclose(struct drm_device *dev)
4811 {
4812         int ret;
4813
4814         if (drm_core_check_feature(dev, DRIVER_MODESET))
4815                 return;
4816
4817         ret = i915_gem_idle(dev);
4818         if (ret)
4819                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4820 }
4821
4822 static void
4823 init_ring_lists(struct intel_ring_buffer *ring)
4824 {
4825         INIT_LIST_HEAD(&ring->active_list);
4826         INIT_LIST_HEAD(&ring->request_list);
4827         INIT_LIST_HEAD(&ring->gpu_write_list);
4828 }
4829
4830 void
4831 i915_gem_load(struct drm_device *dev)
4832 {
4833         int i;
4834         drm_i915_private_t *dev_priv = dev->dev_private;
4835
4836         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4837         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4838         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4839         INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
4840         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4841         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4842         INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
4843         init_ring_lists(&dev_priv->render_ring);
4844         init_ring_lists(&dev_priv->bsd_ring);
4845         init_ring_lists(&dev_priv->blt_ring);
4846         for (i = 0; i < 16; i++)
4847                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4848         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4849                           i915_gem_retire_work_handler);
4850         init_completion(&dev_priv->error_completion);
4851
4852         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4853         if (IS_GEN3(dev)) {
4854                 u32 tmp = I915_READ(MI_ARB_STATE);
4855                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4856                         /* arb state is a masked write, so set bit + bit in mask */
4857                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4858                         I915_WRITE(MI_ARB_STATE, tmp);
4859                 }
4860         }
4861
4862         /* Old X drivers will take 0-2 for front, back, depth buffers */
4863         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4864                 dev_priv->fence_reg_start = 3;
4865
4866         if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4867                 dev_priv->num_fence_regs = 16;
4868         else
4869                 dev_priv->num_fence_regs = 8;
4870
4871         /* Initialize fence registers to zero */
4872         switch (INTEL_INFO(dev)->gen) {
4873         case 6:
4874                 for (i = 0; i < 16; i++)
4875                         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4876                 break;
4877         case 5:
4878         case 4:
4879                 for (i = 0; i < 16; i++)
4880                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4881                 break;
4882         case 3:
4883                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4884                         for (i = 0; i < 8; i++)
4885                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4886         case 2:
4887                 for (i = 0; i < 8; i++)
4888                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4889                 break;
4890         }
4891         i915_gem_detect_bit_6_swizzle(dev);
4892         init_waitqueue_head(&dev_priv->pending_flip_queue);
4893
4894         dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4895         dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4896         register_shrinker(&dev_priv->mm.inactive_shrinker);
4897 }
4898
4899 /*
4900  * Create a physically contiguous memory object for this object
4901  * e.g. for cursor + overlay regs
4902  */
4903 static int i915_gem_init_phys_object(struct drm_device *dev,
4904                                      int id, int size, int align)
4905 {
4906         drm_i915_private_t *dev_priv = dev->dev_private;
4907         struct drm_i915_gem_phys_object *phys_obj;
4908         int ret;
4909
4910         if (dev_priv->mm.phys_objs[id - 1] || !size)
4911                 return 0;
4912
4913         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4914         if (!phys_obj)
4915                 return -ENOMEM;
4916
4917         phys_obj->id = id;
4918
4919         phys_obj->handle = drm_pci_alloc(dev, size, align);
4920         if (!phys_obj->handle) {
4921                 ret = -ENOMEM;
4922                 goto kfree_obj;
4923         }
4924 #ifdef CONFIG_X86
4925         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4926 #endif
4927
4928         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4929
4930         return 0;
4931 kfree_obj:
4932         kfree(phys_obj);
4933         return ret;
4934 }
4935
4936 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4937 {
4938         drm_i915_private_t *dev_priv = dev->dev_private;
4939         struct drm_i915_gem_phys_object *phys_obj;
4940
4941         if (!dev_priv->mm.phys_objs[id - 1])
4942                 return;
4943
4944         phys_obj = dev_priv->mm.phys_objs[id - 1];
4945         if (phys_obj->cur_obj) {
4946                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4947         }
4948
4949 #ifdef CONFIG_X86
4950         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4951 #endif
4952         drm_pci_free(dev, phys_obj->handle);
4953         kfree(phys_obj);
4954         dev_priv->mm.phys_objs[id - 1] = NULL;
4955 }
4956
4957 void i915_gem_free_all_phys_object(struct drm_device *dev)
4958 {
4959         int i;
4960
4961         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4962                 i915_gem_free_phys_object(dev, i);
4963 }
4964
4965 void i915_gem_detach_phys_object(struct drm_device *dev,
4966                                  struct drm_i915_gem_object *obj)
4967 {
4968         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4969         char *vaddr;
4970         int i;
4971         int page_count;
4972
4973         if (!obj->phys_obj)
4974                 return;
4975         vaddr = obj->phys_obj->handle->vaddr;
4976
4977         page_count = obj->base.size / PAGE_SIZE;
4978         for (i = 0; i < page_count; i++) {
4979                 struct page *page = read_cache_page_gfp(mapping, i,
4980                                                         GFP_HIGHUSER | __GFP_RECLAIMABLE);
4981                 if (!IS_ERR(page)) {
4982                         char *dst = kmap_atomic(page);
4983                         memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4984                         kunmap_atomic(dst);
4985
4986                         drm_clflush_pages(&page, 1);
4987
4988                         set_page_dirty(page);
4989                         mark_page_accessed(page);
4990                         page_cache_release(page);
4991                 }
4992         }
4993         intel_gtt_chipset_flush();
4994
4995         obj->phys_obj->cur_obj = NULL;
4996         obj->phys_obj = NULL;
4997 }
4998
4999 int
5000 i915_gem_attach_phys_object(struct drm_device *dev,
5001                             struct drm_i915_gem_object *obj,
5002                             int id,
5003                             int align)
5004 {
5005         struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
5006         drm_i915_private_t *dev_priv = dev->dev_private;
5007         int ret = 0;
5008         int page_count;
5009         int i;
5010
5011         if (id > I915_MAX_PHYS_OBJECT)
5012                 return -EINVAL;
5013
5014         if (obj->phys_obj) {
5015                 if (obj->phys_obj->id == id)
5016                         return 0;
5017                 i915_gem_detach_phys_object(dev, obj);
5018         }
5019
5020         /* create a new object */
5021         if (!dev_priv->mm.phys_objs[id - 1]) {
5022                 ret = i915_gem_init_phys_object(dev, id,
5023                                                 obj->base.size, align);
5024                 if (ret) {
5025                         DRM_ERROR("failed to init phys object %d size: %zu\n",
5026                                   id, obj->base.size);
5027                         return ret;
5028                 }
5029         }
5030
5031         /* bind to the object */
5032         obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
5033         obj->phys_obj->cur_obj = obj;
5034
5035         page_count = obj->base.size / PAGE_SIZE;
5036
5037         for (i = 0; i < page_count; i++) {
5038                 struct page *page;
5039                 char *dst, *src;
5040
5041                 page = read_cache_page_gfp(mapping, i,
5042                                            GFP_HIGHUSER | __GFP_RECLAIMABLE);
5043                 if (IS_ERR(page))
5044                         return PTR_ERR(page);
5045
5046                 src = kmap_atomic(page);
5047                 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
5048                 memcpy(dst, src, PAGE_SIZE);
5049                 kunmap_atomic(src);
5050
5051                 mark_page_accessed(page);
5052                 page_cache_release(page);
5053         }
5054
5055         return 0;
5056 }
5057
5058 static int
5059 i915_gem_phys_pwrite(struct drm_device *dev,
5060                      struct drm_i915_gem_object *obj,
5061                      struct drm_i915_gem_pwrite *args,
5062                      struct drm_file *file_priv)
5063 {
5064         void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
5065         char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
5066
5067         if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
5068                 unsigned long unwritten;
5069
5070                 /* The physical object once assigned is fixed for the lifetime
5071                  * of the obj, so we can safely drop the lock and continue
5072                  * to access vaddr.
5073                  */
5074                 mutex_unlock(&dev->struct_mutex);
5075                 unwritten = copy_from_user(vaddr, user_data, args->size);
5076                 mutex_lock(&dev->struct_mutex);
5077                 if (unwritten)
5078                         return -EFAULT;
5079         }
5080
5081         intel_gtt_chipset_flush();
5082         return 0;
5083 }
5084
5085 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5086 {
5087         struct drm_i915_file_private *file_priv = file->driver_priv;
5088
5089         /* Clean up our request list when the client is going away, so that
5090          * later retire_requests won't dereference our soon-to-be-gone
5091          * file_priv.
5092          */
5093         spin_lock(&file_priv->mm.lock);
5094         while (!list_empty(&file_priv->mm.request_list)) {
5095                 struct drm_i915_gem_request *request;
5096
5097                 request = list_first_entry(&file_priv->mm.request_list,
5098                                            struct drm_i915_gem_request,
5099                                            client_list);
5100                 list_del(&request->client_list);
5101                 request->file_priv = NULL;
5102         }
5103         spin_unlock(&file_priv->mm.lock);
5104 }
5105
5106 static int
5107 i915_gpu_is_active(struct drm_device *dev)
5108 {
5109         drm_i915_private_t *dev_priv = dev->dev_private;
5110         int lists_empty;
5111
5112         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5113                       list_empty(&dev_priv->mm.active_list);
5114
5115         return !lists_empty;
5116 }
5117
5118 static int
5119 i915_gem_inactive_shrink(struct shrinker *shrinker,
5120                          int nr_to_scan,
5121                          gfp_t gfp_mask)
5122 {
5123         struct drm_i915_private *dev_priv =
5124                 container_of(shrinker,
5125                              struct drm_i915_private,
5126                              mm.inactive_shrinker);
5127         struct drm_device *dev = dev_priv->dev;
5128         struct drm_i915_gem_object *obj, *next;
5129         int cnt;
5130
5131         if (!mutex_trylock(&dev->struct_mutex))
5132                 return 0;
5133
5134         /* "fast-path" to count number of available objects */
5135         if (nr_to_scan == 0) {
5136                 cnt = 0;
5137                 list_for_each_entry(obj,
5138                                     &dev_priv->mm.inactive_list,
5139                                     mm_list)
5140                         cnt++;
5141                 mutex_unlock(&dev->struct_mutex);
5142                 return cnt / 100 * sysctl_vfs_cache_pressure;
5143         }
5144
5145 rescan:
5146         /* first scan for clean buffers */
5147         i915_gem_retire_requests(dev);
5148
5149         list_for_each_entry_safe(obj, next,
5150                                  &dev_priv->mm.inactive_list,
5151                                  mm_list) {
5152                 if (i915_gem_object_is_purgeable(obj)) {
5153                         i915_gem_object_unbind(obj);
5154                         if (--nr_to_scan == 0)
5155                                 break;
5156                 }
5157         }
5158
5159         /* second pass, evict/count anything still on the inactive list */
5160         cnt = 0;
5161         list_for_each_entry_safe(obj, next,
5162                                  &dev_priv->mm.inactive_list,
5163                                  mm_list) {
5164                 if (nr_to_scan) {
5165                         i915_gem_object_unbind(obj);
5166                         nr_to_scan--;
5167                 } else
5168                         cnt++;
5169         }
5170
5171         if (nr_to_scan && i915_gpu_is_active(dev)) {
5172                 /*
5173                  * We are desperate for pages, so as a last resort, wait
5174                  * for the GPU to finish and discard whatever we can.
5175                  * This has a dramatic impact to reduce the number of
5176                  * OOM-killer events whilst running the GPU aggressively.
5177                  */
5178                 if (i915_gpu_idle(dev) == 0)
5179                         goto rescan;
5180         }
5181         mutex_unlock(&dev->struct_mutex);
5182         return cnt / 100 * sysctl_vfs_cache_pressure;
5183 }