drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37
  38 #define I915_GEM_GPU_DOMAINS    (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
  39
  40 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  41 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  42 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  43 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  44                                              int write);
  45 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  46                                                      uint64_t offset,
  47                                                      uint64_t size);
  48 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  49 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  50 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  51                                            unsigned alignment);
  52 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  53 static int i915_gem_evict_something(struct drm_device *dev, int min_size);
  54 static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
  55 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  56                                 struct drm_i915_gem_pwrite *args,
  57                                 struct drm_file *file_priv);
  58
  59 static LIST_HEAD(shrink_list);
  60 static DEFINE_SPINLOCK(shrink_list_lock);
  61
  62 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  63                      unsigned long end)
  64 {
  65         drm_i915_private_t *dev_priv = dev->dev_private;
  66
  67         if (start >= end ||
  68             (start & (PAGE_SIZE - 1)) != 0 ||
  69             (end & (PAGE_SIZE - 1)) != 0) {
  70                 return -EINVAL;
  71         }
  72
  73         drm_mm_init(&dev_priv->mm.gtt_space, start,
  74                     end - start);
  75
  76         dev->gtt_total = (uint32_t) (end - start);
  77
  78         return 0;
  79 }
  80
  81 int
  82 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  83                     struct drm_file *file_priv)
  84 {
  85         struct drm_i915_gem_init *args = data;
  86         int ret;
  87
  88         mutex_lock(&dev->struct_mutex);
  89         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  90         mutex_unlock(&dev->struct_mutex);
  91
  92         return ret;
  93 }
  94
  95 int
  96 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  97                             struct drm_file *file_priv)
  98 {
  99         struct drm_i915_gem_get_aperture *args = data;
 100
 101         if (!(dev->driver->driver_features & DRIVER_GEM))
 102                 return -ENODEV;
 103
 104         args->aper_size = dev->gtt_total;
 105         args->aper_available_size = (args->aper_size -
 106                                      atomic_read(&dev->pin_memory));
 107
 108         return 0;
 109 }
 110
 111
 112 /**
 113  * Creates a new mm object and returns a handle to it.
 114  */
 115 int
 116 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 117                       struct drm_file *file_priv)
 118 {
 119         struct drm_i915_gem_create *args = data;
 120         struct drm_gem_object *obj;
 121         int ret;
 122         u32 handle;
 123
 124         args->size = roundup(args->size, PAGE_SIZE);
 125
 126         /* Allocate the new object */
 127         obj = i915_gem_alloc_object(dev, args->size);
 128         if (obj == NULL)
 129                 return -ENOMEM;
 130
 131         ret = drm_gem_handle_create(file_priv, obj, &handle);
 132         drm_gem_object_handle_unreference_unlocked(obj);
 133
 134         if (ret)
 135                 return ret;
 136
 137         args->handle = handle;
 138
 139         return 0;
 140 }
 141
 142 static inline int
 143 fast_shmem_read(struct page **pages,
 144                 loff_t page_base, int page_offset,
 145                 char __user *data,
 146                 int length)
 147 {
 148         char __iomem *vaddr;
 149         int unwritten;
 150
 151         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 152         if (vaddr == NULL)
 153                 return -ENOMEM;
 154         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 155         kunmap_atomic(vaddr, KM_USER0);
 156
 157         if (unwritten)
 158                 return -EFAULT;
 159
 160         return 0;
 161 }
 162
 163 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 164 {
 165         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 166         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 167
 168         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 169                 obj_priv->tiling_mode != I915_TILING_NONE;
 170 }
 171
 172 static inline int
 173 slow_shmem_copy(struct page *dst_page,
 174                 int dst_offset,
 175                 struct page *src_page,
 176                 int src_offset,
 177                 int length)
 178 {
 179         char *dst_vaddr, *src_vaddr;
 180
 181         dst_vaddr = kmap_atomic(dst_page, KM_USER0);
 182         if (dst_vaddr == NULL)
 183                 return -ENOMEM;
 184
 185         src_vaddr = kmap_atomic(src_page, KM_USER1);
 186         if (src_vaddr == NULL) {
 187                 kunmap_atomic(dst_vaddr, KM_USER0);
 188                 return -ENOMEM;
 189         }
 190
 191         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 192
 193         kunmap_atomic(src_vaddr, KM_USER1);
 194         kunmap_atomic(dst_vaddr, KM_USER0);
 195
 196         return 0;
 197 }
 198
 199 static inline int
 200 slow_shmem_bit17_copy(struct page *gpu_page,
 201                       int gpu_offset,
 202                       struct page *cpu_page,
 203                       int cpu_offset,
 204                       int length,
 205                       int is_read)
 206 {
 207         char *gpu_vaddr, *cpu_vaddr;
 208
 209         /* Use the unswizzled path if this page isn't affected. */
 210         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 211                 if (is_read)
 212                         return slow_shmem_copy(cpu_page, cpu_offset,
 213                                                gpu_page, gpu_offset, length);
 214                 else
 215                         return slow_shmem_copy(gpu_page, gpu_offset,
 216                                                cpu_page, cpu_offset, length);
 217         }
 218
 219         gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
 220         if (gpu_vaddr == NULL)
 221                 return -ENOMEM;
 222
 223         cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
 224         if (cpu_vaddr == NULL) {
 225                 kunmap_atomic(gpu_vaddr, KM_USER0);
 226                 return -ENOMEM;
 227         }
 228
 229         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 230          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 231          */
 232         while (length > 0) {
 233                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 234                 int this_length = min(cacheline_end - gpu_offset, length);
 235                 int swizzled_gpu_offset = gpu_offset ^ 64;
 236
 237                 if (is_read) {
 238                         memcpy(cpu_vaddr + cpu_offset,
 239                                gpu_vaddr + swizzled_gpu_offset,
 240                                this_length);
 241                 } else {
 242                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 243                                cpu_vaddr + cpu_offset,
 244                                this_length);
 245                 }
 246                 cpu_offset += this_length;
 247                 gpu_offset += this_length;
 248                 length -= this_length;
 249         }
 250
 251         kunmap_atomic(cpu_vaddr, KM_USER1);
 252         kunmap_atomic(gpu_vaddr, KM_USER0);
 253
 254         return 0;
 255 }
 256
 257 /**
 258  * This is the fast shmem pread path, which attempts to copy_from_user directly
 259  * from the backing pages of the object to the user's address space.  On a
 260  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 261  */
 262 static int
 263 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 264                           struct drm_i915_gem_pread *args,
 265                           struct drm_file *file_priv)
 266 {
 267         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 268         ssize_t remain;
 269         loff_t offset, page_base;
 270         char __user *user_data;
 271         int page_offset, page_length;
 272         int ret;
 273
 274         user_data = (char __user *) (uintptr_t) args->data_ptr;
 275         remain = args->size;
 276
 277         mutex_lock(&dev->struct_mutex);
 278
 279         ret = i915_gem_object_get_pages(obj, 0);
 280         if (ret != 0)
 281                 goto fail_unlock;
 282
 283         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 284                                                         args->size);
 285         if (ret != 0)
 286                 goto fail_put_pages;
 287
 288         obj_priv = to_intel_bo(obj);
 289         offset = args->offset;
 290
 291         while (remain > 0) {
 292                 /* Operation in this page
 293                  *
 294                  * page_base = page offset within aperture
 295                  * page_offset = offset within page
 296                  * page_length = bytes to copy for this page
 297                  */
 298                 page_base = (offset & ~(PAGE_SIZE-1));
 299                 page_offset = offset & (PAGE_SIZE-1);
 300                 page_length = remain;
 301                 if ((page_offset + remain) > PAGE_SIZE)
 302                         page_length = PAGE_SIZE - page_offset;
 303
 304                 ret = fast_shmem_read(obj_priv->pages,
 305                                       page_base, page_offset,
 306                                       user_data, page_length);
 307                 if (ret)
 308                         goto fail_put_pages;
 309
 310                 remain -= page_length;
 311                 user_data += page_length;
 312                 offset += page_length;
 313         }
 314
 315 fail_put_pages:
 316         i915_gem_object_put_pages(obj);
 317 fail_unlock:
 318         mutex_unlock(&dev->struct_mutex);
 319
 320         return ret;
 321 }
 322
 323 static int
 324 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 325 {
 326         int ret;
 327
 328         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 329
 330         /* If we've insufficient memory to map in the pages, attempt
 331          * to make some space by throwing out some old buffers.
 332          */
 333         if (ret == -ENOMEM) {
 334                 struct drm_device *dev = obj->dev;
 335
 336                 ret = i915_gem_evict_something(dev, obj->size);
 337                 if (ret)
 338                         return ret;
 339
 340                 ret = i915_gem_object_get_pages(obj, 0);
 341         }
 342
 343         return ret;
 344 }
 345
 346 /**
 347  * This is the fallback shmem pread path, which allocates temporary storage
 348  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 349  * can copy out of the object's backing pages while holding the struct mutex
 350  * and not take page faults.
 351  */
 352 static int
 353 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 354                           struct drm_i915_gem_pread *args,
 355                           struct drm_file *file_priv)
 356 {
 357         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 358         struct mm_struct *mm = current->mm;
 359         struct page **user_pages;
 360         ssize_t remain;
 361         loff_t offset, pinned_pages, i;
 362         loff_t first_data_page, last_data_page, num_pages;
 363         int shmem_page_index, shmem_page_offset;
 364         int data_page_index,  data_page_offset;
 365         int page_length;
 366         int ret;
 367         uint64_t data_ptr = args->data_ptr;
 368         int do_bit17_swizzling;
 369
 370         remain = args->size;
 371
 372         /* Pin the user pages containing the data.  We can't fault while
 373          * holding the struct mutex, yet we want to hold it while
 374          * dereferencing the user data.
 375          */
 376         first_data_page = data_ptr / PAGE_SIZE;
 377         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 378         num_pages = last_data_page - first_data_page + 1;
 379
 380         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 381         if (user_pages == NULL)
 382                 return -ENOMEM;
 383
 384         down_read(&mm->mmap_sem);
 385         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 386                                       num_pages, 1, 0, user_pages, NULL);
 387         up_read(&mm->mmap_sem);
 388         if (pinned_pages < num_pages) {
 389                 ret = -EFAULT;
 390                 goto fail_put_user_pages;
 391         }
 392
 393         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 394
 395         mutex_lock(&dev->struct_mutex);
 396
 397         ret = i915_gem_object_get_pages_or_evict(obj);
 398         if (ret)
 399                 goto fail_unlock;
 400
 401         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 402                                                         args->size);
 403         if (ret != 0)
 404                 goto fail_put_pages;
 405
 406         obj_priv = to_intel_bo(obj);
 407         offset = args->offset;
 408
 409         while (remain > 0) {
 410                 /* Operation in this page
 411                  *
 412                  * shmem_page_index = page number within shmem file
 413                  * shmem_page_offset = offset within page in shmem file
 414                  * data_page_index = page number in get_user_pages return
 415                  * data_page_offset = offset with data_page_index page.
 416                  * page_length = bytes to copy for this page
 417                  */
 418                 shmem_page_index = offset / PAGE_SIZE;
 419                 shmem_page_offset = offset & ~PAGE_MASK;
 420                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 421                 data_page_offset = data_ptr & ~PAGE_MASK;
 422
 423                 page_length = remain;
 424                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 425                         page_length = PAGE_SIZE - shmem_page_offset;
 426                 if ((data_page_offset + page_length) > PAGE_SIZE)
 427                         page_length = PAGE_SIZE - data_page_offset;
 428
 429                 if (do_bit17_swizzling) {
 430                         ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 431                                                     shmem_page_offset,
 432                                                     user_pages[data_page_index],
 433                                                     data_page_offset,
 434                                                     page_length,
 435                                                     1);
 436                 } else {
 437                         ret = slow_shmem_copy(user_pages[data_page_index],
 438                                               data_page_offset,
 439                                               obj_priv->pages[shmem_page_index],
 440                                               shmem_page_offset,
 441                                               page_length);
 442                 }
 443                 if (ret)
 444                         goto fail_put_pages;
 445
 446                 remain -= page_length;
 447                 data_ptr += page_length;
 448                 offset += page_length;
 449         }
 450
 451 fail_put_pages:
 452         i915_gem_object_put_pages(obj);
 453 fail_unlock:
 454         mutex_unlock(&dev->struct_mutex);
 455 fail_put_user_pages:
 456         for (i = 0; i < pinned_pages; i++) {
 457                 SetPageDirty(user_pages[i]);
 458                 page_cache_release(user_pages[i]);
 459         }
 460         drm_free_large(user_pages);
 461
 462         return ret;
 463 }
 464
 465 /**
 466  * Reads data from the object referenced by handle.
 467  *
 468  * On error, the contents of *data are undefined.
 469  */
 470 int
 471 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 472                      struct drm_file *file_priv)
 473 {
 474         struct drm_i915_gem_pread *args = data;
 475         struct drm_gem_object *obj;
 476         struct drm_i915_gem_object *obj_priv;
 477         int ret;
 478
 479         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 480         if (obj == NULL)
 481                 return -EBADF;
 482         obj_priv = to_intel_bo(obj);
 483
 484         /* Bounds check source.
 485          *
 486          * XXX: This could use review for overflow issues...
 487          */
 488         if (args->offset > obj->size || args->size > obj->size ||
 489             args->offset + args->size > obj->size) {
 490                 drm_gem_object_unreference_unlocked(obj);
 491                 return -EINVAL;
 492         }
 493
 494         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 495                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 496         } else {
 497                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 498                 if (ret != 0)
 499                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 500                                                         file_priv);
 501         }
 502
 503         drm_gem_object_unreference_unlocked(obj);
 504
 505         return ret;
 506 }
 507
 508 /* This is the fast write path which cannot handle
 509  * page faults in the source data
 510  */
 511
 512 static inline int
 513 fast_user_write(struct io_mapping *mapping,
 514                 loff_t page_base, int page_offset,
 515                 char __user *user_data,
 516                 int length)
 517 {
 518         char *vaddr_atomic;
 519         unsigned long unwritten;
 520
 521         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 522         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 523                                                       user_data, length);
 524         io_mapping_unmap_atomic(vaddr_atomic);
 525         if (unwritten)
 526                 return -EFAULT;
 527         return 0;
 528 }
 529
 530 /* Here's the write path which can sleep for
 531  * page faults
 532  */
 533
 534 static inline int
 535 slow_kernel_write(struct io_mapping *mapping,
 536                   loff_t gtt_base, int gtt_offset,
 537                   struct page *user_page, int user_offset,
 538                   int length)
 539 {
 540         char *src_vaddr, *dst_vaddr;
 541         unsigned long unwritten;
 542
 543         dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
 544         src_vaddr = kmap_atomic(user_page, KM_USER1);
 545         unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
 546                                                       src_vaddr + user_offset,
 547                                                       length);
 548         kunmap_atomic(src_vaddr, KM_USER1);
 549         io_mapping_unmap_atomic(dst_vaddr);
 550         if (unwritten)
 551                 return -EFAULT;
 552         return 0;
 553 }
 554
 555 static inline int
 556 fast_shmem_write(struct page **pages,
 557                  loff_t page_base, int page_offset,
 558                  char __user *data,
 559                  int length)
 560 {
 561         char __iomem *vaddr;
 562         unsigned long unwritten;
 563
 564         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 565         if (vaddr == NULL)
 566                 return -ENOMEM;
 567         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 568         kunmap_atomic(vaddr, KM_USER0);
 569
 570         if (unwritten)
 571                 return -EFAULT;
 572         return 0;
 573 }
 574
 575 /**
 576  * This is the fast pwrite path, where we copy the data directly from the
 577  * user into the GTT, uncached.
 578  */
 579 static int
 580 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 581                          struct drm_i915_gem_pwrite *args,
 582                          struct drm_file *file_priv)
 583 {
 584         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 585         drm_i915_private_t *dev_priv = dev->dev_private;
 586         ssize_t remain;
 587         loff_t offset, page_base;
 588         char __user *user_data;
 589         int page_offset, page_length;
 590         int ret;
 591
 592         user_data = (char __user *) (uintptr_t) args->data_ptr;
 593         remain = args->size;
 594         if (!access_ok(VERIFY_READ, user_data, remain))
 595                 return -EFAULT;
 596
 597
 598         mutex_lock(&dev->struct_mutex);
 599         ret = i915_gem_object_pin(obj, 0);
 600         if (ret) {
 601                 mutex_unlock(&dev->struct_mutex);
 602                 return ret;
 603         }
 604         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 605         if (ret)
 606                 goto fail;
 607
 608         obj_priv = to_intel_bo(obj);
 609         offset = obj_priv->gtt_offset + args->offset;
 610
 611         while (remain > 0) {
 612                 /* Operation in this page
 613                  *
 614                  * page_base = page offset within aperture
 615                  * page_offset = offset within page
 616                  * page_length = bytes to copy for this page
 617                  */
 618                 page_base = (offset & ~(PAGE_SIZE-1));
 619                 page_offset = offset & (PAGE_SIZE-1);
 620                 page_length = remain;
 621                 if ((page_offset + remain) > PAGE_SIZE)
 622                         page_length = PAGE_SIZE - page_offset;
 623
 624                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 625                                        page_offset, user_data, page_length);
 626
 627                 /* If we get a fault while copying data, then (presumably) our
 628                  * source page isn't available.  Return the error and we'll
 629                  * retry in the slow path.
 630                  */
 631                 if (ret)
 632                         goto fail;
 633
 634                 remain -= page_length;
 635                 user_data += page_length;
 636                 offset += page_length;
 637         }
 638
 639 fail:
 640         i915_gem_object_unpin(obj);
 641         mutex_unlock(&dev->struct_mutex);
 642
 643         return ret;
 644 }
 645
 646 /**
 647  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 648  * the memory and maps it using kmap_atomic for copying.
 649  *
 650  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 651  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 652  */
 653 static int
 654 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 655                          struct drm_i915_gem_pwrite *args,
 656                          struct drm_file *file_priv)
 657 {
 658         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 659         drm_i915_private_t *dev_priv = dev->dev_private;
 660         ssize_t remain;
 661         loff_t gtt_page_base, offset;
 662         loff_t first_data_page, last_data_page, num_pages;
 663         loff_t pinned_pages, i;
 664         struct page **user_pages;
 665         struct mm_struct *mm = current->mm;
 666         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 667         int ret;
 668         uint64_t data_ptr = args->data_ptr;
 669
 670         remain = args->size;
 671
 672         /* Pin the user pages containing the data.  We can't fault while
 673          * holding the struct mutex, and all of the pwrite implementations
 674          * want to hold it while dereferencing the user data.
 675          */
 676         first_data_page = data_ptr / PAGE_SIZE;
 677         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 678         num_pages = last_data_page - first_data_page + 1;
 679
 680         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 681         if (user_pages == NULL)
 682                 return -ENOMEM;
 683
 684         down_read(&mm->mmap_sem);
 685         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 686                                       num_pages, 0, 0, user_pages, NULL);
 687         up_read(&mm->mmap_sem);
 688         if (pinned_pages < num_pages) {
 689                 ret = -EFAULT;
 690                 goto out_unpin_pages;
 691         }
 692
 693         mutex_lock(&dev->struct_mutex);
 694         ret = i915_gem_object_pin(obj, 0);
 695         if (ret)
 696                 goto out_unlock;
 697
 698         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 699         if (ret)
 700                 goto out_unpin_object;
 701
 702         obj_priv = to_intel_bo(obj);
 703         offset = obj_priv->gtt_offset + args->offset;
 704
 705         while (remain > 0) {
 706                 /* Operation in this page
 707                  *
 708                  * gtt_page_base = page offset within aperture
 709                  * gtt_page_offset = offset within page in aperture
 710                  * data_page_index = page number in get_user_pages return
 711                  * data_page_offset = offset with data_page_index page.
 712                  * page_length = bytes to copy for this page
 713                  */
 714                 gtt_page_base = offset & PAGE_MASK;
 715                 gtt_page_offset = offset & ~PAGE_MASK;
 716                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 717                 data_page_offset = data_ptr & ~PAGE_MASK;
 718
 719                 page_length = remain;
 720                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 721                         page_length = PAGE_SIZE - gtt_page_offset;
 722                 if ((data_page_offset + page_length) > PAGE_SIZE)
 723                         page_length = PAGE_SIZE - data_page_offset;
 724
 725                 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
 726                                         gtt_page_base, gtt_page_offset,
 727                                         user_pages[data_page_index],
 728                                         data_page_offset,
 729                                         page_length);
 730
 731                 /* If we get a fault while copying data, then (presumably) our
 732                  * source page isn't available.  Return the error and we'll
 733                  * retry in the slow path.
 734                  */
 735                 if (ret)
 736                         goto out_unpin_object;
 737
 738                 remain -= page_length;
 739                 offset += page_length;
 740                 data_ptr += page_length;
 741         }
 742
 743 out_unpin_object:
 744         i915_gem_object_unpin(obj);
 745 out_unlock:
 746         mutex_unlock(&dev->struct_mutex);
 747 out_unpin_pages:
 748         for (i = 0; i < pinned_pages; i++)
 749                 page_cache_release(user_pages[i]);
 750         drm_free_large(user_pages);
 751
 752         return ret;
 753 }
 754
 755 /**
 756  * This is the fast shmem pwrite path, which attempts to directly
 757  * copy_from_user into the kmapped pages backing the object.
 758  */
 759 static int
 760 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 761                            struct drm_i915_gem_pwrite *args,
 762                            struct drm_file *file_priv)
 763 {
 764         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 765         ssize_t remain;
 766         loff_t offset, page_base;
 767         char __user *user_data;
 768         int page_offset, page_length;
 769         int ret;
 770
 771         user_data = (char __user *) (uintptr_t) args->data_ptr;
 772         remain = args->size;
 773
 774         mutex_lock(&dev->struct_mutex);
 775
 776         ret = i915_gem_object_get_pages(obj, 0);
 777         if (ret != 0)
 778                 goto fail_unlock;
 779
 780         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 781         if (ret != 0)
 782                 goto fail_put_pages;
 783
 784         obj_priv = to_intel_bo(obj);
 785         offset = args->offset;
 786         obj_priv->dirty = 1;
 787
 788         while (remain > 0) {
 789                 /* Operation in this page
 790                  *
 791                  * page_base = page offset within aperture
 792                  * page_offset = offset within page
 793                  * page_length = bytes to copy for this page
 794                  */
 795                 page_base = (offset & ~(PAGE_SIZE-1));
 796                 page_offset = offset & (PAGE_SIZE-1);
 797                 page_length = remain;
 798                 if ((page_offset + remain) > PAGE_SIZE)
 799                         page_length = PAGE_SIZE - page_offset;
 800
 801                 ret = fast_shmem_write(obj_priv->pages,
 802                                        page_base, page_offset,
 803                                        user_data, page_length);
 804                 if (ret)
 805                         goto fail_put_pages;
 806
 807                 remain -= page_length;
 808                 user_data += page_length;
 809                 offset += page_length;
 810         }
 811
 812 fail_put_pages:
 813         i915_gem_object_put_pages(obj);
 814 fail_unlock:
 815         mutex_unlock(&dev->struct_mutex);
 816
 817         return ret;
 818 }
 819
 820 /**
 821  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 822  * the memory and maps it using kmap_atomic for copying.
 823  *
 824  * This avoids taking mmap_sem for faulting on the user's address while the
 825  * struct_mutex is held.
 826  */
 827 static int
 828 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 829                            struct drm_i915_gem_pwrite *args,
 830                            struct drm_file *file_priv)
 831 {
 832         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 833         struct mm_struct *mm = current->mm;
 834         struct page **user_pages;
 835         ssize_t remain;
 836         loff_t offset, pinned_pages, i;
 837         loff_t first_data_page, last_data_page, num_pages;
 838         int shmem_page_index, shmem_page_offset;
 839         int data_page_index,  data_page_offset;
 840         int page_length;
 841         int ret;
 842         uint64_t data_ptr = args->data_ptr;
 843         int do_bit17_swizzling;
 844
 845         remain = args->size;
 846
 847         /* Pin the user pages containing the data.  We can't fault while
 848          * holding the struct mutex, and all of the pwrite implementations
 849          * want to hold it while dereferencing the user data.
 850          */
 851         first_data_page = data_ptr / PAGE_SIZE;
 852         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 853         num_pages = last_data_page - first_data_page + 1;
 854
 855         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 856         if (user_pages == NULL)
 857                 return -ENOMEM;
 858
 859         down_read(&mm->mmap_sem);
 860         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 861                                       num_pages, 0, 0, user_pages, NULL);
 862         up_read(&mm->mmap_sem);
 863         if (pinned_pages < num_pages) {
 864                 ret = -EFAULT;
 865                 goto fail_put_user_pages;
 866         }
 867
 868         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 869
 870         mutex_lock(&dev->struct_mutex);
 871
 872         ret = i915_gem_object_get_pages_or_evict(obj);
 873         if (ret)
 874                 goto fail_unlock;
 875
 876         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 877         if (ret != 0)
 878                 goto fail_put_pages;
 879
 880         obj_priv = to_intel_bo(obj);
 881         offset = args->offset;
 882         obj_priv->dirty = 1;
 883
 884         while (remain > 0) {
 885                 /* Operation in this page
 886                  *
 887                  * shmem_page_index = page number within shmem file
 888                  * shmem_page_offset = offset within page in shmem file
 889                  * data_page_index = page number in get_user_pages return
 890                  * data_page_offset = offset with data_page_index page.
 891                  * page_length = bytes to copy for this page
 892                  */
 893                 shmem_page_index = offset / PAGE_SIZE;
 894                 shmem_page_offset = offset & ~PAGE_MASK;
 895                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 896                 data_page_offset = data_ptr & ~PAGE_MASK;
 897
 898                 page_length = remain;
 899                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 900                         page_length = PAGE_SIZE - shmem_page_offset;
 901                 if ((data_page_offset + page_length) > PAGE_SIZE)
 902                         page_length = PAGE_SIZE - data_page_offset;
 903
 904                 if (do_bit17_swizzling) {
 905                         ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 906                                                     shmem_page_offset,
 907                                                     user_pages[data_page_index],
 908                                                     data_page_offset,
 909                                                     page_length,
 910                                                     0);
 911                 } else {
 912                         ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
 913                                               shmem_page_offset,
 914                                               user_pages[data_page_index],
 915                                               data_page_offset,
 916                                               page_length);
 917                 }
 918                 if (ret)
 919                         goto fail_put_pages;
 920
 921                 remain -= page_length;
 922                 data_ptr += page_length;
 923                 offset += page_length;
 924         }
 925
 926 fail_put_pages:
 927         i915_gem_object_put_pages(obj);
 928 fail_unlock:
 929         mutex_unlock(&dev->struct_mutex);
 930 fail_put_user_pages:
 931         for (i = 0; i < pinned_pages; i++)
 932                 page_cache_release(user_pages[i]);
 933         drm_free_large(user_pages);
 934
 935         return ret;
 936 }
 937
 938 /**
 939  * Writes data to the object referenced by handle.
 940  *
 941  * On error, the contents of the buffer that were to be modified are undefined.
 942  */
 943 int
 944 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 945                       struct drm_file *file_priv)
 946 {
 947         struct drm_i915_gem_pwrite *args = data;
 948         struct drm_gem_object *obj;
 949         struct drm_i915_gem_object *obj_priv;
 950         int ret = 0;
 951
 952         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 953         if (obj == NULL)
 954                 return -EBADF;
 955         obj_priv = to_intel_bo(obj);
 956
 957         /* Bounds check destination.
 958          *
 959          * XXX: This could use review for overflow issues...
 960          */
 961         if (args->offset > obj->size || args->size > obj->size ||
 962             args->offset + args->size > obj->size) {
 963                 drm_gem_object_unreference_unlocked(obj);
 964                 return -EINVAL;
 965         }
 966
 967         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 968          * it would end up going through the fenced access, and we'll get
 969          * different detiling behavior between reading and writing.
 970          * pread/pwrite currently are reading and writing from the CPU
 971          * perspective, requiring manual detiling by the client.
 972          */
 973         if (obj_priv->phys_obj)
 974                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 975         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 976                  dev->gtt_total != 0) {
 977                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 978                 if (ret == -EFAULT) {
 979                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 980                                                        file_priv);
 981                 }
 982         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 983                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 984         } else {
 985                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 986                 if (ret == -EFAULT) {
 987                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 988                                                          file_priv);
 989                 }
 990         }
 991
 992 #if WATCH_PWRITE
 993         if (ret)
 994                 DRM_INFO("pwrite failed %d\n", ret);
 995 #endif
 996
 997         drm_gem_object_unreference_unlocked(obj);
 998
 999         return ret;
1000 }
1001
1002 /**
1003  * Called when user space prepares to use an object with the CPU, either
1004  * through the mmap ioctl's mapping or a GTT mapping.
1005  */
1006 int
1007 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1008                           struct drm_file *file_priv)
1009 {
1010         struct drm_i915_private *dev_priv = dev->dev_private;
1011         struct drm_i915_gem_set_domain *args = data;
1012         struct drm_gem_object *obj;
1013         struct drm_i915_gem_object *obj_priv;
1014         uint32_t read_domains = args->read_domains;
1015         uint32_t write_domain = args->write_domain;
1016         int ret;
1017
1018         if (!(dev->driver->driver_features & DRIVER_GEM))
1019                 return -ENODEV;
1020
1021         /* Only handle setting domains to types used by the CPU. */
1022         if (write_domain & I915_GEM_GPU_DOMAINS)
1023                 return -EINVAL;
1024
1025         if (read_domains & I915_GEM_GPU_DOMAINS)
1026                 return -EINVAL;
1027
1028         /* Having something in the write domain implies it's in the read
1029          * domain, and only that read domain.  Enforce that in the request.
1030          */
1031         if (write_domain != 0 && read_domains != write_domain)
1032                 return -EINVAL;
1033
1034         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1035         if (obj == NULL)
1036                 return -EBADF;
1037         obj_priv = to_intel_bo(obj);
1038
1039         mutex_lock(&dev->struct_mutex);
1040
1041         intel_mark_busy(dev, obj);
1042
1043 #if WATCH_BUF
1044         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1045                  obj, obj->size, read_domains, write_domain);
1046 #endif
1047         if (read_domains & I915_GEM_DOMAIN_GTT) {
1048                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1049
1050                 /* Update the LRU on the fence for the CPU access that's
1051                  * about to occur.
1052                  */
1053                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1054                         struct drm_i915_fence_reg *reg =
1055                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1056                         list_move_tail(&reg->lru_list,
1057                                        &dev_priv->mm.fence_list);
1058                 }
1059
1060                 /* Silently promote "you're not bound, there was nothing to do"
1061                  * to success, since the client was just asking us to
1062                  * make sure everything was done.
1063                  */
1064                 if (ret == -EINVAL)
1065                         ret = 0;
1066         } else {
1067                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1068         }
1069
1070         drm_gem_object_unreference(obj);
1071         mutex_unlock(&dev->struct_mutex);
1072         return ret;
1073 }
1074
1075 /**
1076  * Called when user space has done writes to this buffer
1077  */
1078 int
1079 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1080                       struct drm_file *file_priv)
1081 {
1082         struct drm_i915_gem_sw_finish *args = data;
1083         struct drm_gem_object *obj;
1084         struct drm_i915_gem_object *obj_priv;
1085         int ret = 0;
1086
1087         if (!(dev->driver->driver_features & DRIVER_GEM))
1088                 return -ENODEV;
1089
1090         mutex_lock(&dev->struct_mutex);
1091         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1092         if (obj == NULL) {
1093                 mutex_unlock(&dev->struct_mutex);
1094                 return -EBADF;
1095         }
1096
1097 #if WATCH_BUF
1098         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1099                  __func__, args->handle, obj, obj->size);
1100 #endif
1101         obj_priv = to_intel_bo(obj);
1102
1103         /* Pinned buffers may be scanout, so flush the cache */
1104         if (obj_priv->pin_count)
1105                 i915_gem_object_flush_cpu_write_domain(obj);
1106
1107         drm_gem_object_unreference(obj);
1108         mutex_unlock(&dev->struct_mutex);
1109         return ret;
1110 }
1111
1112 /**
1113  * Maps the contents of an object, returning the address it is mapped
1114  * into.
1115  *
1116  * While the mapping holds a reference on the contents of the object, it doesn't
1117  * imply a ref on the object itself.
1118  */
1119 int
1120 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1121                    struct drm_file *file_priv)
1122 {
1123         struct drm_i915_gem_mmap *args = data;
1124         struct drm_gem_object *obj;
1125         loff_t offset;
1126         unsigned long addr;
1127
1128         if (!(dev->driver->driver_features & DRIVER_GEM))
1129                 return -ENODEV;
1130
1131         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1132         if (obj == NULL)
1133                 return -EBADF;
1134
1135         offset = args->offset;
1136
1137         down_write(&current->mm->mmap_sem);
1138         addr = do_mmap(obj->filp, 0, args->size,
1139                        PROT_READ | PROT_WRITE, MAP_SHARED,
1140                        args->offset);
1141         up_write(&current->mm->mmap_sem);
1142         drm_gem_object_unreference_unlocked(obj);
1143         if (IS_ERR((void *)addr))
1144                 return addr;
1145
1146         args->addr_ptr = (uint64_t) addr;
1147
1148         return 0;
1149 }
1150
1151 /**
1152  * i915_gem_fault - fault a page into the GTT
1153  * vma: VMA in question
1154  * vmf: fault info
1155  *
1156  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1157  * from userspace.  The fault handler takes care of binding the object to
1158  * the GTT (if needed), allocating and programming a fence register (again,
1159  * only if needed based on whether the old reg is still valid or the object
1160  * is tiled) and inserting a new PTE into the faulting process.
1161  *
1162  * Note that the faulting process may involve evicting existing objects
1163  * from the GTT and/or fence registers to make room.  So performance may
1164  * suffer if the GTT working set is large or there are few fence registers
1165  * left.
1166  */
1167 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1168 {
1169         struct drm_gem_object *obj = vma->vm_private_data;
1170         struct drm_device *dev = obj->dev;
1171         struct drm_i915_private *dev_priv = dev->dev_private;
1172         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1173         pgoff_t page_offset;
1174         unsigned long pfn;
1175         int ret = 0;
1176         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1177
1178         /* We don't use vmf->pgoff since that has the fake offset */
1179         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1180                 PAGE_SHIFT;
1181
1182         /* Now bind it into the GTT if needed */
1183         mutex_lock(&dev->struct_mutex);
1184         if (!obj_priv->gtt_space) {
1185                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1186                 if (ret)
1187                         goto unlock;
1188
1189                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1190
1191                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1192                 if (ret)
1193                         goto unlock;
1194         }
1195
1196         /* Need a new fence register? */
1197         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1198                 ret = i915_gem_object_get_fence_reg(obj);
1199                 if (ret)
1200                         goto unlock;
1201         }
1202
1203         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1204                 page_offset;
1205
1206         /* Finally, remap it using the new GTT offset */
1207         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1208 unlock:
1209         mutex_unlock(&dev->struct_mutex);
1210
1211         switch (ret) {
1212         case 0:
1213         case -ERESTARTSYS:
1214                 return VM_FAULT_NOPAGE;
1215         case -ENOMEM:
1216         case -EAGAIN:
1217                 return VM_FAULT_OOM;
1218         default:
1219                 return VM_FAULT_SIGBUS;
1220         }
1221 }
1222
1223 /**
1224  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1225  * @obj: obj in question
1226  *
1227  * GEM memory mapping works by handing back to userspace a fake mmap offset
1228  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1229  * up the object based on the offset and sets up the various memory mapping
1230  * structures.
1231  *
1232  * This routine allocates and attaches a fake offset for @obj.
1233  */
1234 static int
1235 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1236 {
1237         struct drm_device *dev = obj->dev;
1238         struct drm_gem_mm *mm = dev->mm_private;
1239         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1240         struct drm_map_list *list;
1241         struct drm_local_map *map;
1242         int ret = 0;
1243
1244         /* Set the object up for mmap'ing */
1245         list = &obj->map_list;
1246         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1247         if (!list->map)
1248                 return -ENOMEM;
1249
1250         map = list->map;
1251         map->type = _DRM_GEM;
1252         map->size = obj->size;
1253         map->handle = obj;
1254
1255         /* Get a DRM GEM mmap offset allocated... */
1256         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1257                                                     obj->size / PAGE_SIZE, 0, 0);
1258         if (!list->file_offset_node) {
1259                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1260                 ret = -ENOMEM;
1261                 goto out_free_list;
1262         }
1263
1264         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1265                                                   obj->size / PAGE_SIZE, 0);
1266         if (!list->file_offset_node) {
1267                 ret = -ENOMEM;
1268                 goto out_free_list;
1269         }
1270
1271         list->hash.key = list->file_offset_node->start;
1272         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1273                 DRM_ERROR("failed to add to map hash\n");
1274                 ret = -ENOMEM;
1275                 goto out_free_mm;
1276         }
1277
1278         /* By now we should be all set, any drm_mmap request on the offset
1279          * below will get to our mmap & fault handler */
1280         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1281
1282         return 0;
1283
1284 out_free_mm:
1285         drm_mm_put_block(list->file_offset_node);
1286 out_free_list:
1287         kfree(list->map);
1288
1289         return ret;
1290 }
1291
1292 /**
1293  * i915_gem_release_mmap - remove physical page mappings
1294  * @obj: obj in question
1295  *
1296  * Preserve the reservation of the mmapping with the DRM core code, but
1297  * relinquish ownership of the pages back to the system.
1298  *
1299  * It is vital that we remove the page mapping if we have mapped a tiled
1300  * object through the GTT and then lose the fence register due to
1301  * resource pressure. Similarly if the object has been moved out of the
1302  * aperture, than pages mapped into userspace must be revoked. Removing the
1303  * mapping will then trigger a page fault on the next user access, allowing
1304  * fixup by i915_gem_fault().
1305  */
1306 void
1307 i915_gem_release_mmap(struct drm_gem_object *obj)
1308 {
1309         struct drm_device *dev = obj->dev;
1310         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1311
1312         if (dev->dev_mapping)
1313                 unmap_mapping_range(dev->dev_mapping,
1314                                     obj_priv->mmap_offset, obj->size, 1);
1315 }
1316
1317 static void
1318 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1319 {
1320         struct drm_device *dev = obj->dev;
1321         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1322         struct drm_gem_mm *mm = dev->mm_private;
1323         struct drm_map_list *list;
1324
1325         list = &obj->map_list;
1326         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1327
1328         if (list->file_offset_node) {
1329                 drm_mm_put_block(list->file_offset_node);
1330                 list->file_offset_node = NULL;
1331         }
1332
1333         if (list->map) {
1334                 kfree(list->map);
1335                 list->map = NULL;
1336         }
1337
1338         obj_priv->mmap_offset = 0;
1339 }
1340
1341 /**
1342  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1343  * @obj: object to check
1344  *
1345  * Return the required GTT alignment for an object, taking into account
1346  * potential fence register mapping if needed.
1347  */
1348 static uint32_t
1349 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1350 {
1351         struct drm_device *dev = obj->dev;
1352         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1353         int start, i;
1354
1355         /*
1356          * Minimum alignment is 4k (GTT page size), but might be greater
1357          * if a fence register is needed for the object.
1358          */
1359         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1360                 return 4096;
1361
1362         /*
1363          * Previous chips need to be aligned to the size of the smallest
1364          * fence register that can contain the object.
1365          */
1366         if (IS_I9XX(dev))
1367                 start = 1024*1024;
1368         else
1369                 start = 512*1024;
1370
1371         for (i = start; i < obj->size; i <<= 1)
1372                 ;
1373
1374         return i;
1375 }
1376
1377 /**
1378  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1379  * @dev: DRM device
1380  * @data: GTT mapping ioctl data
1381  * @file_priv: GEM object info
1382  *
1383  * Simply returns the fake offset to userspace so it can mmap it.
1384  * The mmap call will end up in drm_gem_mmap(), which will set things
1385  * up so we can get faults in the handler above.
1386  *
1387  * The fault handler will take care of binding the object into the GTT
1388  * (since it may have been evicted to make room for something), allocating
1389  * a fence register, and mapping the appropriate aperture address into
1390  * userspace.
1391  */
1392 int
1393 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1394                         struct drm_file *file_priv)
1395 {
1396         struct drm_i915_gem_mmap_gtt *args = data;
1397         struct drm_i915_private *dev_priv = dev->dev_private;
1398         struct drm_gem_object *obj;
1399         struct drm_i915_gem_object *obj_priv;
1400         int ret;
1401
1402         if (!(dev->driver->driver_features & DRIVER_GEM))
1403                 return -ENODEV;
1404
1405         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1406         if (obj == NULL)
1407                 return -EBADF;
1408
1409         mutex_lock(&dev->struct_mutex);
1410
1411         obj_priv = to_intel_bo(obj);
1412
1413         if (obj_priv->madv != I915_MADV_WILLNEED) {
1414                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1415                 drm_gem_object_unreference(obj);
1416                 mutex_unlock(&dev->struct_mutex);
1417                 return -EINVAL;
1418         }
1419
1420
1421         if (!obj_priv->mmap_offset) {
1422                 ret = i915_gem_create_mmap_offset(obj);
1423                 if (ret) {
1424                         drm_gem_object_unreference(obj);
1425                         mutex_unlock(&dev->struct_mutex);
1426                         return ret;
1427                 }
1428         }
1429
1430         args->offset = obj_priv->mmap_offset;
1431
1432         /*
1433          * Pull it into the GTT so that we have a page list (makes the
1434          * initial fault faster and any subsequent flushing possible).
1435          */
1436         if (!obj_priv->agp_mem) {
1437                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1438                 if (ret) {
1439                         drm_gem_object_unreference(obj);
1440                         mutex_unlock(&dev->struct_mutex);
1441                         return ret;
1442                 }
1443                 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1444         }
1445
1446         drm_gem_object_unreference(obj);
1447         mutex_unlock(&dev->struct_mutex);
1448
1449         return 0;
1450 }
1451
1452 void
1453 i915_gem_object_put_pages(struct drm_gem_object *obj)
1454 {
1455         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1456         int page_count = obj->size / PAGE_SIZE;
1457         int i;
1458
1459         BUG_ON(obj_priv->pages_refcount == 0);
1460         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1461
1462         if (--obj_priv->pages_refcount != 0)
1463                 return;
1464
1465         if (obj_priv->tiling_mode != I915_TILING_NONE)
1466                 i915_gem_object_save_bit_17_swizzle(obj);
1467
1468         if (obj_priv->madv == I915_MADV_DONTNEED)
1469                 obj_priv->dirty = 0;
1470
1471         for (i = 0; i < page_count; i++) {
1472                 if (obj_priv->dirty)
1473                         set_page_dirty(obj_priv->pages[i]);
1474
1475                 if (obj_priv->madv == I915_MADV_WILLNEED)
1476                         mark_page_accessed(obj_priv->pages[i]);
1477
1478                 page_cache_release(obj_priv->pages[i]);
1479         }
1480         obj_priv->dirty = 0;
1481
1482         drm_free_large(obj_priv->pages);
1483         obj_priv->pages = NULL;
1484 }
1485
1486 static void
1487 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
1488 {
1489         struct drm_device *dev = obj->dev;
1490         drm_i915_private_t *dev_priv = dev->dev_private;
1491         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1492
1493         /* Add a reference if we're newly entering the active list. */
1494         if (!obj_priv->active) {
1495                 drm_gem_object_reference(obj);
1496                 obj_priv->active = 1;
1497         }
1498         /* Move from whatever list we were on to the tail of execution. */
1499         spin_lock(&dev_priv->mm.active_list_lock);
1500         list_move_tail(&obj_priv->list,
1501                        &dev_priv->mm.active_list);
1502         spin_unlock(&dev_priv->mm.active_list_lock);
1503         obj_priv->last_rendering_seqno = seqno;
1504 }
1505
1506 static void
1507 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1508 {
1509         struct drm_device *dev = obj->dev;
1510         drm_i915_private_t *dev_priv = dev->dev_private;
1511         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1512
1513         BUG_ON(!obj_priv->active);
1514         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1515         obj_priv->last_rendering_seqno = 0;
1516 }
1517
1518 /* Immediately discard the backing storage */
1519 static void
1520 i915_gem_object_truncate(struct drm_gem_object *obj)
1521 {
1522         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1523         struct inode *inode;
1524
1525         inode = obj->filp->f_path.dentry->d_inode;
1526         if (inode->i_op->truncate)
1527                 inode->i_op->truncate (inode);
1528
1529         obj_priv->madv = __I915_MADV_PURGED;
1530 }
1531
1532 static inline int
1533 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1534 {
1535         return obj_priv->madv == I915_MADV_DONTNEED;
1536 }
1537
1538 static void
1539 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1540 {
1541         struct drm_device *dev = obj->dev;
1542         drm_i915_private_t *dev_priv = dev->dev_private;
1543         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1544
1545         i915_verify_inactive(dev, __FILE__, __LINE__);
1546         if (obj_priv->pin_count != 0)
1547                 list_del_init(&obj_priv->list);
1548         else
1549                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1550
1551         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1552
1553         obj_priv->last_rendering_seqno = 0;
1554         if (obj_priv->active) {
1555                 obj_priv->active = 0;
1556                 drm_gem_object_unreference(obj);
1557         }
1558         i915_verify_inactive(dev, __FILE__, __LINE__);
1559 }
1560
1561 static void
1562 i915_gem_process_flushing_list(struct drm_device *dev,
1563                                uint32_t flush_domains, uint32_t seqno)
1564 {
1565         drm_i915_private_t *dev_priv = dev->dev_private;
1566         struct drm_i915_gem_object *obj_priv, *next;
1567
1568         list_for_each_entry_safe(obj_priv, next,
1569                                  &dev_priv->mm.gpu_write_list,
1570                                  gpu_write_list) {
1571                 struct drm_gem_object *obj = &obj_priv->base;
1572
1573                 if ((obj->write_domain & flush_domains) ==
1574                     obj->write_domain) {
1575                         uint32_t old_write_domain = obj->write_domain;
1576
1577                         obj->write_domain = 0;
1578                         list_del_init(&obj_priv->gpu_write_list);
1579                         i915_gem_object_move_to_active(obj, seqno);
1580
1581                         /* update the fence lru list */
1582                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1583                                 struct drm_i915_fence_reg *reg =
1584                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1585                                 list_move_tail(&reg->lru_list,
1586                                                 &dev_priv->mm.fence_list);
1587                         }
1588
1589                         trace_i915_gem_object_change_domain(obj,
1590                                                             obj->read_domains,
1591                                                             old_write_domain);
1592                 }
1593         }
1594 }
1595
1596 #define PIPE_CONTROL_FLUSH(addr)                                        \
1597         OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
1598                  PIPE_CONTROL_DEPTH_STALL);                             \
1599         OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
1600         OUT_RING(0);                                                    \
1601         OUT_RING(0);                                                    \
1602
1603 /**
1604  * Creates a new sequence number, emitting a write of it to the status page
1605  * plus an interrupt, which will trigger i915_user_interrupt_handler.
1606  *
1607  * Must be called with struct_lock held.
1608  *
1609  * Returned sequence numbers are nonzero on success.
1610  */
1611 uint32_t
1612 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1613                  uint32_t flush_domains)
1614 {
1615         drm_i915_private_t *dev_priv = dev->dev_private;
1616         struct drm_i915_file_private *i915_file_priv = NULL;
1617         struct drm_i915_gem_request *request;
1618         uint32_t seqno;
1619         int was_empty;
1620         RING_LOCALS;
1621
1622         if (file_priv != NULL)
1623                 i915_file_priv = file_priv->driver_priv;
1624
1625         request = kzalloc(sizeof(*request), GFP_KERNEL);
1626         if (request == NULL)
1627                 return 0;
1628
1629         /* Grab the seqno we're going to make this request be, and bump the
1630          * next (skipping 0 so it can be the reserved no-seqno value).
1631          */
1632         seqno = dev_priv->mm.next_gem_seqno;
1633         dev_priv->mm.next_gem_seqno++;
1634         if (dev_priv->mm.next_gem_seqno == 0)
1635                 dev_priv->mm.next_gem_seqno++;
1636
1637         if (HAS_PIPE_CONTROL(dev)) {
1638                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
1639
1640                 /*
1641                  * Workaround qword write incoherence by flushing the
1642                  * PIPE_NOTIFY buffers out to memory before requesting
1643                  * an interrupt.
1644                  */
1645                 BEGIN_LP_RING(32);
1646                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
1647                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
1648                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
1649                 OUT_RING(seqno);
1650                 OUT_RING(0);
1651                 PIPE_CONTROL_FLUSH(scratch_addr);
1652                 scratch_addr += 128; /* write to separate cachelines */
1653                 PIPE_CONTROL_FLUSH(scratch_addr);
1654                 scratch_addr += 128;
1655                 PIPE_CONTROL_FLUSH(scratch_addr);
1656                 scratch_addr += 128;
1657                 PIPE_CONTROL_FLUSH(scratch_addr);
1658                 scratch_addr += 128;
1659                 PIPE_CONTROL_FLUSH(scratch_addr);
1660                 scratch_addr += 128;
1661                 PIPE_CONTROL_FLUSH(scratch_addr);
1662                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
1663                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
1664                          PIPE_CONTROL_NOTIFY);
1665                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
1666                 OUT_RING(seqno);
1667                 OUT_RING(0);
1668                 ADVANCE_LP_RING();
1669         } else {
1670                 BEGIN_LP_RING(4);
1671                 OUT_RING(MI_STORE_DWORD_INDEX);
1672                 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1673                 OUT_RING(seqno);
1674
1675                 OUT_RING(MI_USER_INTERRUPT);
1676                 ADVANCE_LP_RING();
1677         }
1678
1679         DRM_DEBUG_DRIVER("%d\n", seqno);
1680
1681         request->seqno = seqno;
1682         request->emitted_jiffies = jiffies;
1683         was_empty = list_empty(&dev_priv->mm.request_list);
1684         list_add_tail(&request->list, &dev_priv->mm.request_list);
1685         if (i915_file_priv) {
1686                 list_add_tail(&request->client_list,
1687                               &i915_file_priv->mm.request_list);
1688         } else {
1689                 INIT_LIST_HEAD(&request->client_list);
1690         }
1691
1692         /* Associate any objects on the flushing list matching the write
1693          * domain we're flushing with our flush.
1694          */
1695         if (flush_domains != 0)
1696                 i915_gem_process_flushing_list(dev, flush_domains, seqno);
1697
1698         if (!dev_priv->mm.suspended) {
1699                 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1700                 if (was_empty)
1701                         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1702         }
1703         return seqno;
1704 }
1705
1706 /**
1707  * Command execution barrier
1708  *
1709  * Ensures that all commands in the ring are finished
1710  * before signalling the CPU
1711  */
1712 static uint32_t
1713 i915_retire_commands(struct drm_device *dev)
1714 {
1715         drm_i915_private_t *dev_priv = dev->dev_private;
1716         uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1717         uint32_t flush_domains = 0;
1718         RING_LOCALS;
1719
1720         /* The sampler always gets flushed on i965 (sigh) */
1721         if (IS_I965G(dev))
1722                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1723         BEGIN_LP_RING(2);
1724         OUT_RING(cmd);
1725         OUT_RING(0); /* noop */
1726         ADVANCE_LP_RING();
1727         return flush_domains;
1728 }
1729
1730 /**
1731  * Moves buffers associated only with the given active seqno from the active
1732  * to inactive list, potentially freeing them.
1733  */
1734 static void
1735 i915_gem_retire_request(struct drm_device *dev,
1736                         struct drm_i915_gem_request *request)
1737 {
1738         drm_i915_private_t *dev_priv = dev->dev_private;
1739
1740         trace_i915_gem_request_retire(dev, request->seqno);
1741
1742         /* Move any buffers on the active list that are no longer referenced
1743          * by the ringbuffer to the flushing/inactive lists as appropriate.
1744          */
1745         spin_lock(&dev_priv->mm.active_list_lock);
1746         while (!list_empty(&dev_priv->mm.active_list)) {
1747                 struct drm_gem_object *obj;
1748                 struct drm_i915_gem_object *obj_priv;
1749
1750                 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1751                                             struct drm_i915_gem_object,
1752                                             list);
1753                 obj = &obj_priv->base;
1754
1755                 /* If the seqno being retired doesn't match the oldest in the
1756                  * list, then the oldest in the list must still be newer than
1757                  * this seqno.
1758                  */
1759                 if (obj_priv->last_rendering_seqno != request->seqno)
1760                         goto out;
1761
1762 #if WATCH_LRU
1763                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1764                          __func__, request->seqno, obj);
1765 #endif
1766
1767                 if (obj->write_domain != 0)
1768                         i915_gem_object_move_to_flushing(obj);
1769                 else {
1770                         /* Take a reference on the object so it won't be
1771                          * freed while the spinlock is held.  The list
1772                          * protection for this spinlock is safe when breaking
1773                          * the lock like this since the next thing we do
1774                          * is just get the head of the list again.
1775                          */
1776                         drm_gem_object_reference(obj);
1777                         i915_gem_object_move_to_inactive(obj);
1778                         spin_unlock(&dev_priv->mm.active_list_lock);
1779                         drm_gem_object_unreference(obj);
1780                         spin_lock(&dev_priv->mm.active_list_lock);
1781                 }
1782         }
1783 out:
1784         spin_unlock(&dev_priv->mm.active_list_lock);
1785 }
1786
1787 /**
1788  * Returns true if seq1 is later than seq2.
1789  */
1790 bool
1791 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1792 {
1793         return (int32_t)(seq1 - seq2) >= 0;
1794 }
1795
1796 uint32_t
1797 i915_get_gem_seqno(struct drm_device *dev)
1798 {
1799         drm_i915_private_t *dev_priv = dev->dev_private;
1800
1801         if (HAS_PIPE_CONTROL(dev))
1802                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
1803         else
1804                 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1805 }
1806
1807 /**
1808  * This function clears the request list as sequence numbers are passed.
1809  */
1810 void
1811 i915_gem_retire_requests(struct drm_device *dev)
1812 {
1813         drm_i915_private_t *dev_priv = dev->dev_private;
1814         uint32_t seqno;
1815
1816         if (!dev_priv->hw_status_page || list_empty(&dev_priv->mm.request_list))
1817                 return;
1818
1819         seqno = i915_get_gem_seqno(dev);
1820
1821         while (!list_empty(&dev_priv->mm.request_list)) {
1822                 struct drm_i915_gem_request *request;
1823                 uint32_t retiring_seqno;
1824
1825                 request = list_first_entry(&dev_priv->mm.request_list,
1826                                            struct drm_i915_gem_request,
1827                                            list);
1828                 retiring_seqno = request->seqno;
1829
1830                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1831                     atomic_read(&dev_priv->mm.wedged)) {
1832                         i915_gem_retire_request(dev, request);
1833
1834                         list_del(&request->list);
1835                         list_del(&request->client_list);
1836                         kfree(request);
1837                 } else
1838                         break;
1839         }
1840
1841         if (unlikely (dev_priv->trace_irq_seqno &&
1842                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1843                 i915_user_irq_put(dev);
1844                 dev_priv->trace_irq_seqno = 0;
1845         }
1846 }
1847
1848 void
1849 i915_gem_retire_work_handler(struct work_struct *work)
1850 {
1851         drm_i915_private_t *dev_priv;
1852         struct drm_device *dev;
1853
1854         dev_priv = container_of(work, drm_i915_private_t,
1855                                 mm.retire_work.work);
1856         dev = dev_priv->dev;
1857
1858         mutex_lock(&dev->struct_mutex);
1859         i915_gem_retire_requests(dev);
1860         if (!dev_priv->mm.suspended &&
1861             !list_empty(&dev_priv->mm.request_list))
1862                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1863         mutex_unlock(&dev->struct_mutex);
1864 }
1865
1866 int
1867 i915_do_wait_request(struct drm_device *dev, uint32_t seqno, int interruptible)
1868 {
1869         drm_i915_private_t *dev_priv = dev->dev_private;
1870         u32 ier;
1871         int ret = 0;
1872
1873         BUG_ON(seqno == 0);
1874
1875         if (atomic_read(&dev_priv->mm.wedged))
1876                 return -EIO;
1877
1878         if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
1879                 if (HAS_PCH_SPLIT(dev))
1880                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1881                 else
1882                         ier = I915_READ(IER);
1883                 if (!ier) {
1884                         DRM_ERROR("something (likely vbetool) disabled "
1885                                   "interrupts, re-enabling\n");
1886                         i915_driver_irq_preinstall(dev);
1887                         i915_driver_irq_postinstall(dev);
1888                 }
1889
1890                 trace_i915_gem_request_wait_begin(dev, seqno);
1891
1892                 dev_priv->mm.waiting_gem_seqno = seqno;
1893                 i915_user_irq_get(dev);
1894                 if (interruptible)
1895                         ret = wait_event_interruptible(dev_priv->irq_queue,
1896                                 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1897                                 atomic_read(&dev_priv->mm.wedged));
1898                 else
1899                         wait_event(dev_priv->irq_queue,
1900                                 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1901                                 atomic_read(&dev_priv->mm.wedged));
1902
1903                 i915_user_irq_put(dev);
1904                 dev_priv->mm.waiting_gem_seqno = 0;
1905
1906                 trace_i915_gem_request_wait_end(dev, seqno);
1907         }
1908         if (atomic_read(&dev_priv->mm.wedged))
1909                 ret = -EIO;
1910
1911         if (ret && ret != -ERESTARTSYS)
1912                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1913                           __func__, ret, seqno, i915_get_gem_seqno(dev));
1914
1915         /* Directly dispatch request retiring.  While we have the work queue
1916          * to handle this, the waiter on a request often wants an associated
1917          * buffer to have made it to the inactive list, and we would need
1918          * a separate wait queue to handle that.
1919          */
1920         if (ret == 0)
1921                 i915_gem_retire_requests(dev);
1922
1923         return ret;
1924 }
1925
1926 /**
1927  * Waits for a sequence number to be signaled, and cleans up the
1928  * request and object lists appropriately for that event.
1929  */
1930 static int
1931 i915_wait_request(struct drm_device *dev, uint32_t seqno)
1932 {
1933         return i915_do_wait_request(dev, seqno, 1);
1934 }
1935
1936 static void
1937 i915_gem_flush(struct drm_device *dev,
1938                uint32_t invalidate_domains,
1939                uint32_t flush_domains)
1940 {
1941         drm_i915_private_t *dev_priv = dev->dev_private;
1942         uint32_t cmd;
1943         RING_LOCALS;
1944
1945 #if WATCH_EXEC
1946         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1947                   invalidate_domains, flush_domains);
1948 #endif
1949         trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
1950                                      invalidate_domains, flush_domains);
1951
1952         if (flush_domains & I915_GEM_DOMAIN_CPU)
1953                 drm_agp_chipset_flush(dev);
1954
1955         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
1956                 /*
1957                  * read/write caches:
1958                  *
1959                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1960                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
1961                  * also flushed at 2d versus 3d pipeline switches.
1962                  *
1963                  * read-only caches:
1964                  *
1965                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1966                  * MI_READ_FLUSH is set, and is always flushed on 965.
1967                  *
1968                  * I915_GEM_DOMAIN_COMMAND may not exist?
1969                  *
1970                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1971                  * invalidated when MI_EXE_FLUSH is set.
1972                  *
1973                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1974                  * invalidated with every MI_FLUSH.
1975                  *
1976                  * TLBs:
1977                  *
1978                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1979                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1980                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1981                  * are flushed at any MI_FLUSH.
1982                  */
1983
1984                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1985                 if ((invalidate_domains|flush_domains) &
1986                     I915_GEM_DOMAIN_RENDER)
1987                         cmd &= ~MI_NO_WRITE_FLUSH;
1988                 if (!IS_I965G(dev)) {
1989                         /*
1990                          * On the 965, the sampler cache always gets flushed
1991                          * and this bit is reserved.
1992                          */
1993                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1994                                 cmd |= MI_READ_FLUSH;
1995                 }
1996                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1997                         cmd |= MI_EXE_FLUSH;
1998
1999 #if WATCH_EXEC
2000                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
2001 #endif
2002                 BEGIN_LP_RING(2);
2003                 OUT_RING(cmd);
2004                 OUT_RING(MI_NOOP);
2005                 ADVANCE_LP_RING();
2006         }
2007 }
2008
2009 /**
2010  * Ensures that all rendering to the object has completed and the object is
2011  * safe to unbind from the GTT or access from the CPU.
2012  */
2013 static int
2014 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
2015 {
2016         struct drm_device *dev = obj->dev;
2017         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2018         int ret;
2019
2020         /* This function only exists to support waiting for existing rendering,
2021          * not for emitting required flushes.
2022          */
2023         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
2024
2025         /* If there is rendering queued on the buffer being evicted, wait for
2026          * it.
2027          */
2028         if (obj_priv->active) {
2029 #if WATCH_BUF
2030                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2031                           __func__, obj, obj_priv->last_rendering_seqno);
2032 #endif
2033                 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
2034                 if (ret != 0)
2035                         return ret;
2036         }
2037
2038         return 0;
2039 }
2040
2041 /**
2042  * Unbinds an object from the GTT aperture.
2043  */
2044 int
2045 i915_gem_object_unbind(struct drm_gem_object *obj)
2046 {
2047         struct drm_device *dev = obj->dev;
2048         drm_i915_private_t *dev_priv = dev->dev_private;
2049         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2050         int ret = 0;
2051
2052 #if WATCH_BUF
2053         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
2054         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2055 #endif
2056         if (obj_priv->gtt_space == NULL)
2057                 return 0;
2058
2059         if (obj_priv->pin_count != 0) {
2060                 DRM_ERROR("Attempting to unbind pinned buffer\n");
2061                 return -EINVAL;
2062         }
2063
2064         /* blow away mappings if mapped through GTT */
2065         i915_gem_release_mmap(obj);
2066
2067         /* Move the object to the CPU domain to ensure that
2068          * any possible CPU writes while it's not in the GTT
2069          * are flushed when we go to remap it. This will
2070          * also ensure that all pending GPU writes are finished
2071          * before we unbind.
2072          */
2073         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2074         if (ret) {
2075                 if (ret != -ERESTARTSYS)
2076                         DRM_ERROR("set_domain failed: %d\n", ret);
2077                 return ret;
2078         }
2079
2080         BUG_ON(obj_priv->active);
2081
2082         /* release the fence reg _after_ flushing */
2083         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2084                 i915_gem_clear_fence_reg(obj);
2085
2086         if (obj_priv->agp_mem != NULL) {
2087                 drm_unbind_agp(obj_priv->agp_mem);
2088                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2089                 obj_priv->agp_mem = NULL;
2090         }
2091
2092         i915_gem_object_put_pages(obj);
2093         BUG_ON(obj_priv->pages_refcount);
2094
2095         if (obj_priv->gtt_space) {
2096                 atomic_dec(&dev->gtt_count);
2097                 atomic_sub(obj->size, &dev->gtt_memory);
2098
2099                 drm_mm_put_block(obj_priv->gtt_space);
2100                 obj_priv->gtt_space = NULL;
2101         }
2102
2103         /* Remove ourselves from the LRU list if present. */
2104         spin_lock(&dev_priv->mm.active_list_lock);
2105         if (!list_empty(&obj_priv->list))
2106                 list_del_init(&obj_priv->list);
2107         spin_unlock(&dev_priv->mm.active_list_lock);
2108
2109         if (i915_gem_object_is_purgeable(obj_priv))
2110                 i915_gem_object_truncate(obj);
2111
2112         trace_i915_gem_object_unbind(obj);
2113
2114         return 0;
2115 }
2116
2117 static struct drm_gem_object *
2118 i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2119 {
2120         drm_i915_private_t *dev_priv = dev->dev_private;
2121         struct drm_i915_gem_object *obj_priv;
2122         struct drm_gem_object *best = NULL;
2123         struct drm_gem_object *first = NULL;
2124
2125         /* Try to find the smallest clean object */
2126         list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
2127                 struct drm_gem_object *obj = &obj_priv->base;
2128                 if (obj->size >= min_size) {
2129                         if ((!obj_priv->dirty ||
2130                              i915_gem_object_is_purgeable(obj_priv)) &&
2131                             (!best || obj->size < best->size)) {
2132                                 best = obj;
2133                                 if (best->size == min_size)
2134                                         return best;
2135                         }
2136                         if (!first)
2137                             first = obj;
2138                 }
2139         }
2140
2141         return best ? best : first;
2142 }
2143
2144 static int
2145 i915_gpu_idle(struct drm_device *dev)
2146 {
2147         drm_i915_private_t *dev_priv = dev->dev_private;
2148         bool lists_empty;
2149         uint32_t seqno;
2150
2151         spin_lock(&dev_priv->mm.active_list_lock);
2152         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
2153                       list_empty(&dev_priv->mm.active_list);
2154         spin_unlock(&dev_priv->mm.active_list_lock);
2155
2156         if (lists_empty)
2157                 return 0;
2158
2159         /* Flush everything onto the inactive list. */
2160         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2161         seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
2162         if (seqno == 0)
2163                 return -ENOMEM;
2164
2165         return i915_wait_request(dev, seqno);
2166 }
2167
2168 static int
2169 i915_gem_evict_everything(struct drm_device *dev)
2170 {
2171         drm_i915_private_t *dev_priv = dev->dev_private;
2172         int ret;
2173         bool lists_empty;
2174
2175         spin_lock(&dev_priv->mm.active_list_lock);
2176         lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2177                        list_empty(&dev_priv->mm.flushing_list) &&
2178                        list_empty(&dev_priv->mm.active_list));
2179         spin_unlock(&dev_priv->mm.active_list_lock);
2180
2181         if (lists_empty)
2182                 return -ENOSPC;
2183
2184         /* Flush everything (on to the inactive lists) and evict */
2185         ret = i915_gpu_idle(dev);
2186         if (ret)
2187                 return ret;
2188
2189         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2190
2191         ret = i915_gem_evict_from_inactive_list(dev);
2192         if (ret)
2193                 return ret;
2194
2195         spin_lock(&dev_priv->mm.active_list_lock);
2196         lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2197                        list_empty(&dev_priv->mm.flushing_list) &&
2198                        list_empty(&dev_priv->mm.active_list));
2199         spin_unlock(&dev_priv->mm.active_list_lock);
2200         BUG_ON(!lists_empty);
2201
2202         return 0;
2203 }
2204
2205 static int
2206 i915_gem_evict_something(struct drm_device *dev, int min_size)
2207 {
2208         drm_i915_private_t *dev_priv = dev->dev_private;
2209         struct drm_gem_object *obj;
2210         int ret;
2211
2212         for (;;) {
2213                 i915_gem_retire_requests(dev);
2214
2215                 /* If there's an inactive buffer available now, grab it
2216                  * and be done.
2217                  */
2218                 obj = i915_gem_find_inactive_object(dev, min_size);
2219                 if (obj) {
2220                         struct drm_i915_gem_object *obj_priv;
2221
2222 #if WATCH_LRU
2223                         DRM_INFO("%s: evicting %p\n", __func__, obj);
2224 #endif
2225                         obj_priv = to_intel_bo(obj);
2226                         BUG_ON(obj_priv->pin_count != 0);
2227                         BUG_ON(obj_priv->active);
2228
2229                         /* Wait on the rendering and unbind the buffer. */
2230                         return i915_gem_object_unbind(obj);
2231                 }
2232
2233                 /* If we didn't get anything, but the ring is still processing
2234                  * things, wait for the next to finish and hopefully leave us
2235                  * a buffer to evict.
2236                  */
2237                 if (!list_empty(&dev_priv->mm.request_list)) {
2238                         struct drm_i915_gem_request *request;
2239
2240                         request = list_first_entry(&dev_priv->mm.request_list,
2241                                                    struct drm_i915_gem_request,
2242                                                    list);
2243
2244                         ret = i915_wait_request(dev, request->seqno);
2245                         if (ret)
2246                                 return ret;
2247
2248                         continue;
2249                 }
2250
2251                 /* If we didn't have anything on the request list but there
2252                  * are buffers awaiting a flush, emit one and try again.
2253                  * When we wait on it, those buffers waiting for that flush
2254                  * will get moved to inactive.
2255                  */
2256                 if (!list_empty(&dev_priv->mm.flushing_list)) {
2257                         struct drm_i915_gem_object *obj_priv;
2258
2259                         /* Find an object that we can immediately reuse */
2260                         list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
2261                                 obj = &obj_priv->base;
2262                                 if (obj->size >= min_size)
2263                                         break;
2264
2265                                 obj = NULL;
2266                         }
2267
2268                         if (obj != NULL) {
2269                                 uint32_t seqno;
2270
2271                                 i915_gem_flush(dev,
2272                                                obj->write_domain,
2273                                                obj->write_domain);
2274                                 seqno = i915_add_request(dev, NULL, obj->write_domain);
2275                                 if (seqno == 0)
2276                                         return -ENOMEM;
2277                                 continue;
2278                         }
2279                 }
2280
2281                 /* If we didn't do any of the above, there's no single buffer
2282                  * large enough to swap out for the new one, so just evict
2283                  * everything and start again. (This should be rare.)
2284                  */
2285                 if (!list_empty (&dev_priv->mm.inactive_list))
2286                         return i915_gem_evict_from_inactive_list(dev);
2287                 else
2288                         return i915_gem_evict_everything(dev);
2289         }
2290 }
2291
2292 int
2293 i915_gem_object_get_pages(struct drm_gem_object *obj,
2294                           gfp_t gfpmask)
2295 {
2296         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2297         int page_count, i;
2298         struct address_space *mapping;
2299         struct inode *inode;
2300         struct page *page;
2301
2302         if (obj_priv->pages_refcount++ != 0)
2303                 return 0;
2304
2305         /* Get the list of pages out of our struct file.  They'll be pinned
2306          * at this point until we release them.
2307          */
2308         page_count = obj->size / PAGE_SIZE;
2309         BUG_ON(obj_priv->pages != NULL);
2310         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2311         if (obj_priv->pages == NULL) {
2312                 obj_priv->pages_refcount--;
2313                 return -ENOMEM;
2314         }
2315
2316         inode = obj->filp->f_path.dentry->d_inode;
2317         mapping = inode->i_mapping;
2318         for (i = 0; i < page_count; i++) {
2319                 page = read_cache_page_gfp(mapping, i,
2320                                            mapping_gfp_mask (mapping) |
2321                                            __GFP_COLD |
2322                                            gfpmask);
2323                 if (IS_ERR(page))
2324                         goto err_pages;
2325
2326                 obj_priv->pages[i] = page;
2327         }
2328
2329         if (obj_priv->tiling_mode != I915_TILING_NONE)
2330                 i915_gem_object_do_bit_17_swizzle(obj);
2331
2332         return 0;
2333
2334 err_pages:
2335         while (i--)
2336                 page_cache_release(obj_priv->pages[i]);
2337
2338         drm_free_large(obj_priv->pages);
2339         obj_priv->pages = NULL;
2340         obj_priv->pages_refcount--;
2341         return PTR_ERR(page);
2342 }
2343
2344 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2345 {
2346         struct drm_gem_object *obj = reg->obj;
2347         struct drm_device *dev = obj->dev;
2348         drm_i915_private_t *dev_priv = dev->dev_private;
2349         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2350         int regnum = obj_priv->fence_reg;
2351         uint64_t val;
2352
2353         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2354                     0xfffff000) << 32;
2355         val |= obj_priv->gtt_offset & 0xfffff000;
2356         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2357                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2358
2359         if (obj_priv->tiling_mode == I915_TILING_Y)
2360                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2361         val |= I965_FENCE_REG_VALID;
2362
2363         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2364 }
2365
2366 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2367 {
2368         struct drm_gem_object *obj = reg->obj;
2369         struct drm_device *dev = obj->dev;
2370         drm_i915_private_t *dev_priv = dev->dev_private;
2371         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2372         int regnum = obj_priv->fence_reg;
2373         uint64_t val;
2374
2375         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2376                     0xfffff000) << 32;
2377         val |= obj_priv->gtt_offset & 0xfffff000;
2378         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2379         if (obj_priv->tiling_mode == I915_TILING_Y)
2380                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2381         val |= I965_FENCE_REG_VALID;
2382
2383         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2384 }
2385
2386 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2387 {
2388         struct drm_gem_object *obj = reg->obj;
2389         struct drm_device *dev = obj->dev;
2390         drm_i915_private_t *dev_priv = dev->dev_private;
2391         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2392         int regnum = obj_priv->fence_reg;
2393         int tile_width;
2394         uint32_t fence_reg, val;
2395         uint32_t pitch_val;
2396
2397         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2398             (obj_priv->gtt_offset & (obj->size - 1))) {
2399                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2400                      __func__, obj_priv->gtt_offset, obj->size);
2401                 return;
2402         }
2403
2404         if (obj_priv->tiling_mode == I915_TILING_Y &&
2405             HAS_128_BYTE_Y_TILING(dev))
2406                 tile_width = 128;
2407         else
2408                 tile_width = 512;
2409
2410         /* Note: pitch better be a power of two tile widths */
2411         pitch_val = obj_priv->stride / tile_width;
2412         pitch_val = ffs(pitch_val) - 1;
2413
2414         if (obj_priv->tiling_mode == I915_TILING_Y &&
2415             HAS_128_BYTE_Y_TILING(dev))
2416                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2417         else
2418                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2419
2420         val = obj_priv->gtt_offset;
2421         if (obj_priv->tiling_mode == I915_TILING_Y)
2422                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2423         val |= I915_FENCE_SIZE_BITS(obj->size);
2424         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2425         val |= I830_FENCE_REG_VALID;
2426
2427         if (regnum < 8)
2428                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2429         else
2430                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2431         I915_WRITE(fence_reg, val);
2432 }
2433
2434 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2435 {
2436         struct drm_gem_object *obj = reg->obj;
2437         struct drm_device *dev = obj->dev;
2438         drm_i915_private_t *dev_priv = dev->dev_private;
2439         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2440         int regnum = obj_priv->fence_reg;
2441         uint32_t val;
2442         uint32_t pitch_val;
2443         uint32_t fence_size_bits;
2444
2445         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2446             (obj_priv->gtt_offset & (obj->size - 1))) {
2447                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2448                      __func__, obj_priv->gtt_offset);
2449                 return;
2450         }
2451
2452         pitch_val = obj_priv->stride / 128;
2453         pitch_val = ffs(pitch_val) - 1;
2454         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2455
2456         val = obj_priv->gtt_offset;
2457         if (obj_priv->tiling_mode == I915_TILING_Y)
2458                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2459         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2460         WARN_ON(fence_size_bits & ~0x00000f00);
2461         val |= fence_size_bits;
2462         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2463         val |= I830_FENCE_REG_VALID;
2464
2465         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2466 }
2467
2468 static int i915_find_fence_reg(struct drm_device *dev)
2469 {
2470         struct drm_i915_fence_reg *reg = NULL;
2471         struct drm_i915_gem_object *obj_priv = NULL;
2472         struct drm_i915_private *dev_priv = dev->dev_private;
2473         struct drm_gem_object *obj = NULL;
2474         int i, avail, ret;
2475
2476         /* First try to find a free reg */
2477         avail = 0;
2478         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2479                 reg = &dev_priv->fence_regs[i];
2480                 if (!reg->obj)
2481                         return i;
2482
2483                 obj_priv = to_intel_bo(reg->obj);
2484                 if (!obj_priv->pin_count)
2485                     avail++;
2486         }
2487
2488         if (avail == 0)
2489                 return -ENOSPC;
2490
2491         /* None available, try to steal one or wait for a user to finish */
2492         i = I915_FENCE_REG_NONE;
2493         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2494                             lru_list) {
2495                 obj = reg->obj;
2496                 obj_priv = to_intel_bo(obj);
2497
2498                 if (obj_priv->pin_count)
2499                         continue;
2500
2501                 /* found one! */
2502                 i = obj_priv->fence_reg;
2503                 break;
2504         }
2505
2506         BUG_ON(i == I915_FENCE_REG_NONE);
2507
2508         /* We only have a reference on obj from the active list. put_fence_reg
2509          * might drop that one, causing a use-after-free in it. So hold a
2510          * private reference to obj like the other callers of put_fence_reg
2511          * (set_tiling ioctl) do. */
2512         drm_gem_object_reference(obj);
2513         ret = i915_gem_object_put_fence_reg(obj);
2514         drm_gem_object_unreference(obj);
2515         if (ret != 0)
2516                 return ret;
2517
2518         return i;
2519 }
2520
2521 /**
2522  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2523  * @obj: object to map through a fence reg
2524  *
2525  * When mapping objects through the GTT, userspace wants to be able to write
2526  * to them without having to worry about swizzling if the object is tiled.
2527  *
2528  * This function walks the fence regs looking for a free one for @obj,
2529  * stealing one if it can't find any.
2530  *
2531  * It then sets up the reg based on the object's properties: address, pitch
2532  * and tiling format.
2533  */
2534 int
2535 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2536 {
2537         struct drm_device *dev = obj->dev;
2538         struct drm_i915_private *dev_priv = dev->dev_private;
2539         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2540         struct drm_i915_fence_reg *reg = NULL;
2541         int ret;
2542
2543         /* Just update our place in the LRU if our fence is getting used. */
2544         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2545                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2546                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2547                 return 0;
2548         }
2549
2550         switch (obj_priv->tiling_mode) {
2551         case I915_TILING_NONE:
2552                 WARN(1, "allocating a fence for non-tiled object?\n");
2553                 break;
2554         case I915_TILING_X:
2555                 if (!obj_priv->stride)
2556                         return -EINVAL;
2557                 WARN((obj_priv->stride & (512 - 1)),
2558                      "object 0x%08x is X tiled but has non-512B pitch\n",
2559                      obj_priv->gtt_offset);
2560                 break;
2561         case I915_TILING_Y:
2562                 if (!obj_priv->stride)
2563                         return -EINVAL;
2564                 WARN((obj_priv->stride & (128 - 1)),
2565                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2566                      obj_priv->gtt_offset);
2567                 break;
2568         }
2569
2570         ret = i915_find_fence_reg(dev);
2571         if (ret < 0)
2572                 return ret;
2573
2574         obj_priv->fence_reg = ret;
2575         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2576         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2577
2578         reg->obj = obj;
2579
2580         if (IS_GEN6(dev))
2581                 sandybridge_write_fence_reg(reg);
2582         else if (IS_I965G(dev))
2583                 i965_write_fence_reg(reg);
2584         else if (IS_I9XX(dev))
2585                 i915_write_fence_reg(reg);
2586         else
2587                 i830_write_fence_reg(reg);
2588
2589         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2590                         obj_priv->tiling_mode);
2591
2592         return 0;
2593 }
2594
2595 /**
2596  * i915_gem_clear_fence_reg - clear out fence register info
2597  * @obj: object to clear
2598  *
2599  * Zeroes out the fence register itself and clears out the associated
2600  * data structures in dev_priv and obj_priv.
2601  */
2602 static void
2603 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2604 {
2605         struct drm_device *dev = obj->dev;
2606         drm_i915_private_t *dev_priv = dev->dev_private;
2607         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2608         struct drm_i915_fence_reg *reg =
2609                 &dev_priv->fence_regs[obj_priv->fence_reg];
2610
2611         if (IS_GEN6(dev)) {
2612                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2613                              (obj_priv->fence_reg * 8), 0);
2614         } else if (IS_I965G(dev)) {
2615                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2616         } else {
2617                 uint32_t fence_reg;
2618
2619                 if (obj_priv->fence_reg < 8)
2620                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2621                 else
2622                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2623                                                        8) * 4;
2624
2625                 I915_WRITE(fence_reg, 0);
2626         }
2627
2628         reg->obj = NULL;
2629         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2630         list_del_init(&reg->lru_list);
2631 }
2632
2633 /**
2634  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2635  * to the buffer to finish, and then resets the fence register.
2636  * @obj: tiled object holding a fence register.
2637  *
2638  * Zeroes out the fence register itself and clears out the associated
2639  * data structures in dev_priv and obj_priv.
2640  */
2641 int
2642 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2643 {
2644         struct drm_device *dev = obj->dev;
2645         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2646
2647         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2648                 return 0;
2649
2650         /* If we've changed tiling, GTT-mappings of the object
2651          * need to re-fault to ensure that the correct fence register
2652          * setup is in place.
2653          */
2654         i915_gem_release_mmap(obj);
2655
2656         /* On the i915, GPU access to tiled buffers is via a fence,
2657          * therefore we must wait for any outstanding access to complete
2658          * before clearing the fence.
2659          */
2660         if (!IS_I965G(dev)) {
2661                 int ret;
2662
2663                 i915_gem_object_flush_gpu_write_domain(obj);
2664                 ret = i915_gem_object_wait_rendering(obj);
2665                 if (ret != 0)
2666                         return ret;
2667         }
2668
2669         i915_gem_object_flush_gtt_write_domain(obj);
2670         i915_gem_clear_fence_reg (obj);
2671
2672         return 0;
2673 }
2674
2675 /**
2676  * Finds free space in the GTT aperture and binds the object there.
2677  */
2678 static int
2679 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2680 {
2681         struct drm_device *dev = obj->dev;
2682         drm_i915_private_t *dev_priv = dev->dev_private;
2683         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2684         struct drm_mm_node *free_space;
2685         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2686         int ret;
2687
2688         if (obj_priv->madv != I915_MADV_WILLNEED) {
2689                 DRM_ERROR("Attempting to bind a purgeable object\n");
2690                 return -EINVAL;
2691         }
2692
2693         if (alignment == 0)
2694                 alignment = i915_gem_get_gtt_alignment(obj);
2695         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2696                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2697                 return -EINVAL;
2698         }
2699
2700  search_free:
2701         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2702                                         obj->size, alignment, 0);
2703         if (free_space != NULL) {
2704                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2705                                                        alignment);
2706                 if (obj_priv->gtt_space != NULL) {
2707                         obj_priv->gtt_space->private = obj;
2708                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2709                 }
2710         }
2711         if (obj_priv->gtt_space == NULL) {
2712                 /* If the gtt is empty and we're still having trouble
2713                  * fitting our object in, we're out of memory.
2714                  */
2715 #if WATCH_LRU
2716                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2717 #endif
2718                 ret = i915_gem_evict_something(dev, obj->size);
2719                 if (ret)
2720                         return ret;
2721
2722                 goto search_free;
2723         }
2724
2725 #if WATCH_BUF
2726         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2727                  obj->size, obj_priv->gtt_offset);
2728 #endif
2729         ret = i915_gem_object_get_pages(obj, gfpmask);
2730         if (ret) {
2731                 drm_mm_put_block(obj_priv->gtt_space);
2732                 obj_priv->gtt_space = NULL;
2733
2734                 if (ret == -ENOMEM) {
2735                         /* first try to clear up some space from the GTT */
2736                         ret = i915_gem_evict_something(dev, obj->size);
2737                         if (ret) {
2738                                 /* now try to shrink everyone else */
2739                                 if (gfpmask) {
2740                                         gfpmask = 0;
2741                                         goto search_free;
2742                                 }
2743
2744                                 return ret;
2745                         }
2746
2747                         goto search_free;
2748                 }
2749
2750                 return ret;
2751         }
2752
2753         /* Create an AGP memory structure pointing at our pages, and bind it
2754          * into the GTT.
2755          */
2756         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2757                                                obj_priv->pages,
2758                                                obj->size >> PAGE_SHIFT,
2759                                                obj_priv->gtt_offset,
2760                                                obj_priv->agp_type);
2761         if (obj_priv->agp_mem == NULL) {
2762                 i915_gem_object_put_pages(obj);
2763                 drm_mm_put_block(obj_priv->gtt_space);
2764                 obj_priv->gtt_space = NULL;
2765
2766                 ret = i915_gem_evict_something(dev, obj->size);
2767                 if (ret)
2768                         return ret;
2769
2770                 goto search_free;
2771         }
2772         atomic_inc(&dev->gtt_count);
2773         atomic_add(obj->size, &dev->gtt_memory);
2774
2775         /* Assert that the object is not currently in any GPU domain. As it
2776          * wasn't in the GTT, there shouldn't be any way it could have been in
2777          * a GPU cache
2778          */
2779         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2780         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2781
2782         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2783
2784         return 0;
2785 }
2786
2787 void
2788 i915_gem_clflush_object(struct drm_gem_object *obj)
2789 {
2790         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2791
2792         /* If we don't have a page list set up, then we're not pinned
2793          * to GPU, and we can ignore the cache flush because it'll happen
2794          * again at bind time.
2795          */
2796         if (obj_priv->pages == NULL)
2797                 return;
2798
2799         trace_i915_gem_object_clflush(obj);
2800
2801         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2802 }
2803
2804 /** Flushes any GPU write domain for the object if it's dirty. */
2805 static void
2806 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2807 {
2808         struct drm_device *dev = obj->dev;
2809         uint32_t old_write_domain;
2810
2811         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2812                 return;
2813
2814         /* Queue the GPU write cache flushing we need. */
2815         old_write_domain = obj->write_domain;
2816         i915_gem_flush(dev, 0, obj->write_domain);
2817         (void) i915_add_request(dev, NULL, obj->write_domain);
2818         BUG_ON(obj->write_domain);
2819
2820         trace_i915_gem_object_change_domain(obj,
2821                                             obj->read_domains,
2822                                             old_write_domain);
2823 }
2824
2825 /** Flushes the GTT write domain for the object if it's dirty. */
2826 static void
2827 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2828 {
2829         uint32_t old_write_domain;
2830
2831         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2832                 return;
2833
2834         /* No actual flushing is required for the GTT write domain.   Writes
2835          * to it immediately go to main memory as far as we know, so there's
2836          * no chipset flush.  It also doesn't land in render cache.
2837          */
2838         old_write_domain = obj->write_domain;
2839         obj->write_domain = 0;
2840
2841         trace_i915_gem_object_change_domain(obj,
2842                                             obj->read_domains,
2843                                             old_write_domain);
2844 }
2845
2846 /** Flushes the CPU write domain for the object if it's dirty. */
2847 static void
2848 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2849 {
2850         struct drm_device *dev = obj->dev;
2851         uint32_t old_write_domain;
2852
2853         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2854                 return;
2855
2856         i915_gem_clflush_object(obj);
2857         drm_agp_chipset_flush(dev);
2858         old_write_domain = obj->write_domain;
2859         obj->write_domain = 0;
2860
2861         trace_i915_gem_object_change_domain(obj,
2862                                             obj->read_domains,
2863                                             old_write_domain);
2864 }
2865
2866 void
2867 i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2868 {
2869         switch (obj->write_domain) {
2870         case I915_GEM_DOMAIN_GTT:
2871                 i915_gem_object_flush_gtt_write_domain(obj);
2872                 break;
2873         case I915_GEM_DOMAIN_CPU:
2874                 i915_gem_object_flush_cpu_write_domain(obj);
2875                 break;
2876         default:
2877                 i915_gem_object_flush_gpu_write_domain(obj);
2878                 break;
2879         }
2880 }
2881
2882 /**
2883  * Moves a single object to the GTT read, and possibly write domain.
2884  *
2885  * This function returns when the move is complete, including waiting on
2886  * flushes to occur.
2887  */
2888 int
2889 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2890 {
2891         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2892         uint32_t old_write_domain, old_read_domains;
2893         int ret;
2894
2895         /* Not valid to be called on unbound objects. */
2896         if (obj_priv->gtt_space == NULL)
2897                 return -EINVAL;
2898
2899         i915_gem_object_flush_gpu_write_domain(obj);
2900         /* Wait on any GPU rendering and flushing to occur. */
2901         ret = i915_gem_object_wait_rendering(obj);
2902         if (ret != 0)
2903                 return ret;
2904
2905         old_write_domain = obj->write_domain;
2906         old_read_domains = obj->read_domains;
2907
2908         /* If we're writing through the GTT domain, then CPU and GPU caches
2909          * will need to be invalidated at next use.
2910          */
2911         if (write)
2912                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2913
2914         i915_gem_object_flush_cpu_write_domain(obj);
2915
2916         /* It should now be out of any other write domains, and we can update
2917          * the domain values for our changes.
2918          */
2919         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2920         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2921         if (write) {
2922                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2923                 obj_priv->dirty = 1;
2924         }
2925
2926         trace_i915_gem_object_change_domain(obj,
2927                                             old_read_domains,
2928                                             old_write_domain);
2929
2930         return 0;
2931 }
2932
2933 /*
2934  * Prepare buffer for display plane. Use uninterruptible for possible flush
2935  * wait, as in modesetting process we're not supposed to be interrupted.
2936  */
2937 int
2938 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2939 {
2940         struct drm_device *dev = obj->dev;
2941         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2942         uint32_t old_write_domain, old_read_domains;
2943         int ret;
2944
2945         /* Not valid to be called on unbound objects. */
2946         if (obj_priv->gtt_space == NULL)
2947                 return -EINVAL;
2948
2949         i915_gem_object_flush_gpu_write_domain(obj);
2950
2951         /* Wait on any GPU rendering and flushing to occur. */
2952         if (obj_priv->active) {
2953 #if WATCH_BUF
2954                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2955                           __func__, obj, obj_priv->last_rendering_seqno);
2956 #endif
2957                 ret = i915_do_wait_request(dev, obj_priv->last_rendering_seqno, 0);
2958                 if (ret != 0)
2959                         return ret;
2960         }
2961
2962         old_write_domain = obj->write_domain;
2963         old_read_domains = obj->read_domains;
2964
2965         obj->read_domains &= I915_GEM_DOMAIN_GTT;
2966
2967         i915_gem_object_flush_cpu_write_domain(obj);
2968
2969         /* It should now be out of any other write domains, and we can update
2970          * the domain values for our changes.
2971          */
2972         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2973         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2974         obj->write_domain = I915_GEM_DOMAIN_GTT;
2975         obj_priv->dirty = 1;
2976
2977         trace_i915_gem_object_change_domain(obj,
2978                                             old_read_domains,
2979                                             old_write_domain);
2980
2981         return 0;
2982 }
2983
2984 /**
2985  * Moves a single object to the CPU read, and possibly write domain.
2986  *
2987  * This function returns when the move is complete, including waiting on
2988  * flushes to occur.
2989  */
2990 static int
2991 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2992 {
2993         uint32_t old_write_domain, old_read_domains;
2994         int ret;
2995
2996         i915_gem_object_flush_gpu_write_domain(obj);
2997         /* Wait on any GPU rendering and flushing to occur. */
2998         ret = i915_gem_object_wait_rendering(obj);
2999         if (ret != 0)
3000                 return ret;
3001
3002         i915_gem_object_flush_gtt_write_domain(obj);
3003
3004         /* If we have a partially-valid cache of the object in the CPU,
3005          * finish invalidating it and free the per-page flags.
3006          */
3007         i915_gem_object_set_to_full_cpu_read_domain(obj);
3008
3009         old_write_domain = obj->write_domain;
3010         old_read_domains = obj->read_domains;
3011
3012         /* Flush the CPU cache if it's still invalid. */
3013         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3014                 i915_gem_clflush_object(obj);
3015
3016                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3017         }
3018
3019         /* It should now be out of any other write domains, and we can update
3020          * the domain values for our changes.
3021          */
3022         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3023
3024         /* If we're writing through the CPU, then the GPU read domains will
3025          * need to be invalidated at next use.
3026          */
3027         if (write) {
3028                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
3029                 obj->write_domain = I915_GEM_DOMAIN_CPU;
3030         }
3031
3032         trace_i915_gem_object_change_domain(obj,
3033                                             old_read_domains,
3034                                             old_write_domain);
3035
3036         return 0;
3037 }
3038
3039 /*
3040  * Set the next domain for the specified object. This
3041  * may not actually perform the necessary flushing/invaliding though,
3042  * as that may want to be batched with other set_domain operations
3043  *
3044  * This is (we hope) the only really tricky part of gem. The goal
3045  * is fairly simple -- track which caches hold bits of the object
3046  * and make sure they remain coherent. A few concrete examples may
3047  * help to explain how it works. For shorthand, we use the notation
3048  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3049  * a pair of read and write domain masks.
3050  *
3051  * Case 1: the batch buffer
3052  *
3053  *      1. Allocated
3054  *      2. Written by CPU
3055  *      3. Mapped to GTT
3056  *      4. Read by GPU
3057  *      5. Unmapped from GTT
3058  *      6. Freed
3059  *
3060  *      Let's take these a step at a time
3061  *
3062  *      1. Allocated
3063  *              Pages allocated from the kernel may still have
3064  *              cache contents, so we set them to (CPU, CPU) always.
3065  *      2. Written by CPU (using pwrite)
3066  *              The pwrite function calls set_domain (CPU, CPU) and
3067  *              this function does nothing (as nothing changes)
3068  *      3. Mapped by GTT
3069  *              This function asserts that the object is not
3070  *              currently in any GPU-based read or write domains
3071  *      4. Read by GPU
3072  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
3073  *              As write_domain is zero, this function adds in the
3074  *              current read domains (CPU+COMMAND, 0).
3075  *              flush_domains is set to CPU.
3076  *              invalidate_domains is set to COMMAND
3077  *              clflush is run to get data out of the CPU caches
3078  *              then i915_dev_set_domain calls i915_gem_flush to
3079  *              emit an MI_FLUSH and drm_agp_chipset_flush
3080  *      5. Unmapped from GTT
3081  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
3082  *              flush_domains and invalidate_domains end up both zero
3083  *              so no flushing/invalidating happens
3084  *      6. Freed
3085  *              yay, done
3086  *
3087  * Case 2: The shared render buffer
3088  *
3089  *      1. Allocated
3090  *      2. Mapped to GTT
3091  *      3. Read/written by GPU
3092  *      4. set_domain to (CPU,CPU)
3093  *      5. Read/written by CPU
3094  *      6. Read/written by GPU
3095  *
3096  *      1. Allocated
3097  *              Same as last example, (CPU, CPU)
3098  *      2. Mapped to GTT
3099  *              Nothing changes (assertions find that it is not in the GPU)
3100  *      3. Read/written by GPU
3101  *              execbuffer calls set_domain (RENDER, RENDER)
3102  *              flush_domains gets CPU
3103  *              invalidate_domains gets GPU
3104  *              clflush (obj)
3105  *              MI_FLUSH and drm_agp_chipset_flush
3106  *      4. set_domain (CPU, CPU)
3107  *              flush_domains gets GPU
3108  *              invalidate_domains gets CPU
3109  *              wait_rendering (obj) to make sure all drawing is complete.
3110  *              This will include an MI_FLUSH to get the data from GPU
3111  *              to memory
3112  *              clflush (obj) to invalidate the CPU cache
3113  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3114  *      5. Read/written by CPU
3115  *              cache lines are loaded and dirtied
3116  *      6. Read written by GPU
3117  *              Same as last GPU access
3118  *
3119  * Case 3: The constant buffer
3120  *
3121  *      1. Allocated
3122  *      2. Written by CPU
3123  *      3. Read by GPU
3124  *      4. Updated (written) by CPU again
3125  *      5. Read by GPU
3126  *
3127  *      1. Allocated
3128  *              (CPU, CPU)
3129  *      2. Written by CPU
3130  *              (CPU, CPU)
3131  *      3. Read by GPU
3132  *              (CPU+RENDER, 0)
3133  *              flush_domains = CPU
3134  *              invalidate_domains = RENDER
3135  *              clflush (obj)
3136  *              MI_FLUSH
3137  *              drm_agp_chipset_flush
3138  *      4. Updated (written) by CPU again
3139  *              (CPU, CPU)
3140  *              flush_domains = 0 (no previous write domain)
3141  *              invalidate_domains = 0 (no new read domains)
3142  *      5. Read by GPU
3143  *              (CPU+RENDER, 0)
3144  *              flush_domains = CPU
3145  *              invalidate_domains = RENDER
3146  *              clflush (obj)
3147  *              MI_FLUSH
3148  *              drm_agp_chipset_flush
3149  */
3150 static void
3151 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
3152 {
3153         struct drm_device               *dev = obj->dev;
3154         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
3155         uint32_t                        invalidate_domains = 0;
3156         uint32_t                        flush_domains = 0;
3157         uint32_t                        old_read_domains;
3158
3159         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3160         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
3161
3162         intel_mark_busy(dev, obj);
3163
3164 #if WATCH_BUF
3165         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3166                  __func__, obj,
3167                  obj->read_domains, obj->pending_read_domains,
3168                  obj->write_domain, obj->pending_write_domain);
3169 #endif
3170         /*
3171          * If the object isn't moving to a new write domain,
3172          * let the object stay in multiple read domains
3173          */
3174         if (obj->pending_write_domain == 0)
3175                 obj->pending_read_domains |= obj->read_domains;
3176         else
3177                 obj_priv->dirty = 1;
3178
3179         /*
3180          * Flush the current write domain if
3181          * the new read domains don't match. Invalidate
3182          * any read domains which differ from the old
3183          * write domain
3184          */
3185         if (obj->write_domain &&
3186             obj->write_domain != obj->pending_read_domains) {
3187                 flush_domains |= obj->write_domain;
3188                 invalidate_domains |=
3189                         obj->pending_read_domains & ~obj->write_domain;
3190         }
3191         /*
3192          * Invalidate any read caches which may have
3193          * stale data. That is, any new read domains.
3194          */
3195         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3196         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3197 #if WATCH_BUF
3198                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3199                          __func__, flush_domains, invalidate_domains);
3200 #endif
3201                 i915_gem_clflush_object(obj);
3202         }
3203
3204         old_read_domains = obj->read_domains;
3205
3206         /* The actual obj->write_domain will be updated with
3207          * pending_write_domain after we emit the accumulated flush for all
3208          * of our domain changes in execbuffers (which clears objects'
3209          * write_domains).  So if we have a current write domain that we
3210          * aren't changing, set pending_write_domain to that.
3211          */
3212         if (flush_domains == 0 && obj->pending_write_domain == 0)
3213                 obj->pending_write_domain = obj->write_domain;
3214         obj->read_domains = obj->pending_read_domains;
3215
3216         dev->invalidate_domains |= invalidate_domains;
3217         dev->flush_domains |= flush_domains;
3218 #if WATCH_BUF
3219         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3220                  __func__,
3221                  obj->read_domains, obj->write_domain,
3222                  dev->invalidate_domains, dev->flush_domains);
3223 #endif
3224
3225         trace_i915_gem_object_change_domain(obj,
3226                                             old_read_domains,
3227                                             obj->write_domain);
3228 }
3229
3230 /**
3231  * Moves the object from a partially CPU read to a full one.
3232  *
3233  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3234  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3235  */
3236 static void
3237 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3238 {
3239         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3240
3241         if (!obj_priv->page_cpu_valid)
3242                 return;
3243
3244         /* If we're partially in the CPU read domain, finish moving it in.
3245          */
3246         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3247                 int i;
3248
3249                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3250                         if (obj_priv->page_cpu_valid[i])
3251                                 continue;
3252                         drm_clflush_pages(obj_priv->pages + i, 1);
3253                 }
3254         }
3255
3256         /* Free the page_cpu_valid mappings which are now stale, whether
3257          * or not we've got I915_GEM_DOMAIN_CPU.
3258          */
3259         kfree(obj_priv->page_cpu_valid);
3260         obj_priv->page_cpu_valid = NULL;
3261 }
3262
3263 /**
3264  * Set the CPU read domain on a range of the object.
3265  *
3266  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3267  * not entirely valid.  The page_cpu_valid member of the object flags which
3268  * pages have been flushed, and will be respected by
3269  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3270  * of the whole object.
3271  *
3272  * This function returns when the move is complete, including waiting on
3273  * flushes to occur.
3274  */
3275 static int
3276 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3277                                           uint64_t offset, uint64_t size)
3278 {
3279         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3280         uint32_t old_read_domains;
3281         int i, ret;
3282
3283         if (offset == 0 && size == obj->size)
3284                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3285
3286         i915_gem_object_flush_gpu_write_domain(obj);
3287         /* Wait on any GPU rendering and flushing to occur. */
3288         ret = i915_gem_object_wait_rendering(obj);
3289         if (ret != 0)
3290                 return ret;
3291         i915_gem_object_flush_gtt_write_domain(obj);
3292
3293         /* If we're already fully in the CPU read domain, we're done. */
3294         if (obj_priv->page_cpu_valid == NULL &&
3295             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3296                 return 0;
3297
3298         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3299          * newly adding I915_GEM_DOMAIN_CPU
3300          */
3301         if (obj_priv->page_cpu_valid == NULL) {
3302                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3303                                                    GFP_KERNEL);
3304                 if (obj_priv->page_cpu_valid == NULL)
3305                         return -ENOMEM;
3306         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3307                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3308
3309         /* Flush the cache on any pages that are still invalid from the CPU's
3310          * perspective.
3311          */
3312         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3313              i++) {
3314                 if (obj_priv->page_cpu_valid[i])
3315                         continue;
3316
3317                 drm_clflush_pages(obj_priv->pages + i, 1);
3318
3319                 obj_priv->page_cpu_valid[i] = 1;
3320         }
3321
3322         /* It should now be out of any other write domains, and we can update
3323          * the domain values for our changes.
3324          */
3325         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3326
3327         old_read_domains = obj->read_domains;
3328         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3329
3330         trace_i915_gem_object_change_domain(obj,
3331                                             old_read_domains,
3332                                             obj->write_domain);
3333
3334         return 0;
3335 }
3336
3337 /**
3338  * Pin an object to the GTT and evaluate the relocations landing in it.
3339  */
3340 static int
3341 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3342                                  struct drm_file *file_priv,
3343                                  struct drm_i915_gem_exec_object2 *entry,
3344                                  struct drm_i915_gem_relocation_entry *relocs)
3345 {
3346         struct drm_device *dev = obj->dev;
3347         drm_i915_private_t *dev_priv = dev->dev_private;
3348         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3349         int i, ret;
3350         void __iomem *reloc_page;
3351         bool need_fence;
3352
3353         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3354                      obj_priv->tiling_mode != I915_TILING_NONE;
3355
3356         /* Check fence reg constraints and rebind if necessary */
3357         if (need_fence && !i915_gem_object_fence_offset_ok(obj,
3358             obj_priv->tiling_mode))
3359                 i915_gem_object_unbind(obj);
3360
3361         /* Choose the GTT offset for our buffer and put it there. */
3362         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3363         if (ret)
3364                 return ret;
3365
3366         /*
3367          * Pre-965 chips need a fence register set up in order to
3368          * properly handle blits to/from tiled surfaces.
3369          */
3370         if (need_fence) {
3371                 ret = i915_gem_object_get_fence_reg(obj);
3372                 if (ret != 0) {
3373                         if (ret != -EBUSY && ret != -ERESTARTSYS)
3374                                 DRM_ERROR("Failure to install fence: %d\n",
3375                                           ret);
3376                         i915_gem_object_unpin(obj);
3377                         return ret;
3378                 }
3379         }
3380
3381         entry->offset = obj_priv->gtt_offset;
3382
3383         /* Apply the relocations, using the GTT aperture to avoid cache
3384          * flushing requirements.
3385          */
3386         for (i = 0; i < entry->relocation_count; i++) {
3387                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3388                 struct drm_gem_object *target_obj;
3389                 struct drm_i915_gem_object *target_obj_priv;
3390                 uint32_t reloc_val, reloc_offset;
3391                 uint32_t __iomem *reloc_entry;
3392
3393                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3394                                                    reloc->target_handle);
3395                 if (target_obj == NULL) {
3396                         i915_gem_object_unpin(obj);
3397                         return -EBADF;
3398                 }
3399                 target_obj_priv = to_intel_bo(target_obj);
3400
3401 #if WATCH_RELOC
3402                 DRM_INFO("%s: obj %p offset %08x target %d "
3403                          "read %08x write %08x gtt %08x "
3404                          "presumed %08x delta %08x\n",
3405                          __func__,
3406                          obj,
3407                          (int) reloc->offset,
3408                          (int) reloc->target_handle,
3409                          (int) reloc->read_domains,
3410                          (int) reloc->write_domain,
3411                          (int) target_obj_priv->gtt_offset,
3412                          (int) reloc->presumed_offset,
3413                          reloc->delta);
3414 #endif
3415
3416                 /* The target buffer should have appeared before us in the
3417                  * exec_object list, so it should have a GTT space bound by now.
3418                  */
3419                 if (target_obj_priv->gtt_space == NULL) {
3420                         DRM_ERROR("No GTT space found for object %d\n",
3421                                   reloc->target_handle);
3422                         drm_gem_object_unreference(target_obj);
3423                         i915_gem_object_unpin(obj);
3424                         return -EINVAL;
3425                 }
3426
3427                 /* Validate that the target is in a valid r/w GPU domain */
3428                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3429                         DRM_ERROR("reloc with multiple write domains: "
3430                                   "obj %p target %d offset %d "
3431                                   "read %08x write %08x",
3432                                   obj, reloc->target_handle,
3433                                   (int) reloc->offset,
3434                                   reloc->read_domains,
3435                                   reloc->write_domain);
3436                         return -EINVAL;
3437                 }
3438                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3439                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3440                         DRM_ERROR("reloc with read/write CPU domains: "
3441                                   "obj %p target %d offset %d "
3442                                   "read %08x write %08x",
3443                                   obj, reloc->target_handle,
3444                                   (int) reloc->offset,
3445                                   reloc->read_domains,
3446                                   reloc->write_domain);
3447                         drm_gem_object_unreference(target_obj);
3448                         i915_gem_object_unpin(obj);
3449                         return -EINVAL;
3450                 }
3451                 if (reloc->write_domain && target_obj->pending_write_domain &&
3452                     reloc->write_domain != target_obj->pending_write_domain) {
3453                         DRM_ERROR("Write domain conflict: "
3454                                   "obj %p target %d offset %d "
3455                                   "new %08x old %08x\n",
3456                                   obj, reloc->target_handle,
3457                                   (int) reloc->offset,
3458                                   reloc->write_domain,
3459                                   target_obj->pending_write_domain);
3460                         drm_gem_object_unreference(target_obj);
3461                         i915_gem_object_unpin(obj);
3462                         return -EINVAL;
3463                 }
3464
3465                 target_obj->pending_read_domains |= reloc->read_domains;
3466                 target_obj->pending_write_domain |= reloc->write_domain;
3467
3468                 /* If the relocation already has the right value in it, no
3469                  * more work needs to be done.
3470                  */
3471                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3472                         drm_gem_object_unreference(target_obj);
3473                         continue;
3474                 }
3475
3476                 /* Check that the relocation address is valid... */
3477                 if (reloc->offset > obj->size - 4) {
3478                         DRM_ERROR("Relocation beyond object bounds: "
3479                                   "obj %p target %d offset %d size %d.\n",
3480                                   obj, reloc->target_handle,
3481                                   (int) reloc->offset, (int) obj->size);
3482                         drm_gem_object_unreference(target_obj);
3483                         i915_gem_object_unpin(obj);
3484                         return -EINVAL;
3485                 }
3486                 if (reloc->offset & 3) {
3487                         DRM_ERROR("Relocation not 4-byte aligned: "
3488                                   "obj %p target %d offset %d.\n",
3489                                   obj, reloc->target_handle,
3490                                   (int) reloc->offset);
3491                         drm_gem_object_unreference(target_obj);
3492                         i915_gem_object_unpin(obj);
3493                         return -EINVAL;
3494                 }
3495
3496                 /* and points to somewhere within the target object. */
3497                 if (reloc->delta >= target_obj->size) {
3498                         DRM_ERROR("Relocation beyond target object bounds: "
3499                                   "obj %p target %d delta %d size %d.\n",
3500                                   obj, reloc->target_handle,
3501                                   (int) reloc->delta, (int) target_obj->size);
3502                         drm_gem_object_unreference(target_obj);
3503                         i915_gem_object_unpin(obj);
3504                         return -EINVAL;
3505                 }
3506
3507                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3508                 if (ret != 0) {
3509                         drm_gem_object_unreference(target_obj);
3510                         i915_gem_object_unpin(obj);
3511                         return -EINVAL;
3512                 }
3513
3514                 /* Map the page containing the relocation we're going to
3515                  * perform.
3516                  */
3517                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3518                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3519                                                       (reloc_offset &
3520                                                        ~(PAGE_SIZE - 1)));
3521                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3522                                                    (reloc_offset & (PAGE_SIZE - 1)));
3523                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3524
3525 #if WATCH_BUF
3526                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3527                           obj, (unsigned int) reloc->offset,
3528                           readl(reloc_entry), reloc_val);
3529 #endif
3530                 writel(reloc_val, reloc_entry);
3531                 io_mapping_unmap_atomic(reloc_page);
3532
3533                 /* The updated presumed offset for this entry will be
3534                  * copied back out to the user.
3535                  */
3536                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3537
3538                 drm_gem_object_unreference(target_obj);
3539         }
3540
3541 #if WATCH_BUF
3542         if (0)
3543                 i915_gem_dump_object(obj, 128, __func__, ~0);
3544 #endif
3545         return 0;
3546 }
3547
3548 /** Dispatch a batchbuffer to the ring
3549  */
3550 static int
3551 i915_dispatch_gem_execbuffer(struct drm_device *dev,
3552                               struct drm_i915_gem_execbuffer2 *exec,
3553                               struct drm_clip_rect *cliprects,
3554                               uint64_t exec_offset)
3555 {
3556         drm_i915_private_t *dev_priv = dev->dev_private;
3557         int nbox = exec->num_cliprects;
3558         int i = 0, count;
3559         uint32_t exec_start, exec_len;
3560         RING_LOCALS;
3561
3562         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3563         exec_len = (uint32_t) exec->batch_len;
3564
3565         trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
3566
3567         count = nbox ? nbox : 1;
3568
3569         for (i = 0; i < count; i++) {
3570                 if (i < nbox) {
3571                         int ret = i915_emit_box(dev, cliprects, i,
3572                                                 exec->DR1, exec->DR4);
3573                         if (ret)
3574                                 return ret;
3575                 }
3576
3577                 if (IS_I830(dev) || IS_845G(dev)) {
3578                         BEGIN_LP_RING(4);
3579                         OUT_RING(MI_BATCH_BUFFER);
3580                         OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3581                         OUT_RING(exec_start + exec_len - 4);
3582                         OUT_RING(0);
3583                         ADVANCE_LP_RING();
3584                 } else {
3585                         BEGIN_LP_RING(2);
3586                         if (IS_I965G(dev)) {
3587                                 OUT_RING(MI_BATCH_BUFFER_START |
3588                                          (2 << 6) |
3589                                          MI_BATCH_NON_SECURE_I965);
3590                                 OUT_RING(exec_start);
3591                         } else {
3592                                 OUT_RING(MI_BATCH_BUFFER_START |
3593                                          (2 << 6));
3594                                 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3595                         }
3596                         ADVANCE_LP_RING();
3597                 }
3598         }
3599
3600         /* XXX breadcrumb */
3601         return 0;
3602 }
3603
3604 /* Throttle our rendering by waiting until the ring has completed our requests
3605  * emitted over 20 msec ago.
3606  *
3607  * Note that if we were to use the current jiffies each time around the loop,
3608  * we wouldn't escape the function with any frames outstanding if the time to
3609  * render a frame was over 20ms.
3610  *
3611  * This should get us reasonable parallelism between CPU and GPU but also
3612  * relatively low latency when blocking on a particular request to finish.
3613  */
3614 static int
3615 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3616 {
3617         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3618         int ret = 0;
3619         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3620
3621         mutex_lock(&dev->struct_mutex);
3622         while (!list_empty(&i915_file_priv->mm.request_list)) {
3623                 struct drm_i915_gem_request *request;
3624
3625                 request = list_first_entry(&i915_file_priv->mm.request_list,
3626                                            struct drm_i915_gem_request,
3627                                            client_list);
3628
3629                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3630                         break;
3631
3632                 ret = i915_wait_request(dev, request->seqno);
3633                 if (ret != 0)
3634                         break;
3635         }
3636         mutex_unlock(&dev->struct_mutex);
3637
3638         return ret;
3639 }
3640
3641 static int
3642 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3643                               uint32_t buffer_count,
3644                               struct drm_i915_gem_relocation_entry **relocs)
3645 {
3646         uint32_t reloc_count = 0, reloc_index = 0, i;
3647         int ret;
3648
3649         *relocs = NULL;
3650         for (i = 0; i < buffer_count; i++) {
3651                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3652                         return -EINVAL;
3653                 reloc_count += exec_list[i].relocation_count;
3654         }
3655
3656         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3657         if (*relocs == NULL) {
3658                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3659                 return -ENOMEM;
3660         }
3661
3662         for (i = 0; i < buffer_count; i++) {
3663                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3664
3665                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3666
3667                 ret = copy_from_user(&(*relocs)[reloc_index],
3668                                      user_relocs,
3669                                      exec_list[i].relocation_count *
3670                                      sizeof(**relocs));
3671                 if (ret != 0) {
3672                         drm_free_large(*relocs);
3673                         *relocs = NULL;
3674                         return -EFAULT;
3675                 }
3676
3677                 reloc_index += exec_list[i].relocation_count;
3678         }
3679
3680         return 0;
3681 }
3682
3683 static int
3684 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3685                             uint32_t buffer_count,
3686                             struct drm_i915_gem_relocation_entry *relocs)
3687 {
3688         uint32_t reloc_count = 0, i;
3689         int ret = 0;
3690
3691         if (relocs == NULL)
3692             return 0;
3693
3694         for (i = 0; i < buffer_count; i++) {
3695                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3696                 int unwritten;
3697
3698                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3699
3700                 unwritten = copy_to_user(user_relocs,
3701                                          &relocs[reloc_count],
3702                                          exec_list[i].relocation_count *
3703                                          sizeof(*relocs));
3704
3705                 if (unwritten) {
3706                         ret = -EFAULT;
3707                         goto err;
3708                 }
3709
3710                 reloc_count += exec_list[i].relocation_count;
3711         }
3712
3713 err:
3714         drm_free_large(relocs);
3715
3716         return ret;
3717 }
3718
3719 static int
3720 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3721                            uint64_t exec_offset)
3722 {
3723         uint32_t exec_start, exec_len;
3724
3725         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3726         exec_len = (uint32_t) exec->batch_len;
3727
3728         if ((exec_start | exec_len) & 0x7)
3729                 return -EINVAL;
3730
3731         if (!exec_start)
3732                 return -EINVAL;
3733
3734         return 0;
3735 }
3736
3737 static int
3738 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3739                                struct drm_gem_object **object_list,
3740                                int count)
3741 {
3742         drm_i915_private_t *dev_priv = dev->dev_private;
3743         struct drm_i915_gem_object *obj_priv;
3744         DEFINE_WAIT(wait);
3745         int i, ret = 0;
3746
3747         for (;;) {
3748                 prepare_to_wait(&dev_priv->pending_flip_queue,
3749                                 &wait, TASK_INTERRUPTIBLE);
3750                 for (i = 0; i < count; i++) {
3751                         obj_priv = to_intel_bo(object_list[i]);
3752                         if (atomic_read(&obj_priv->pending_flip) > 0)
3753                                 break;
3754                 }
3755                 if (i == count)
3756                         break;
3757
3758                 if (!signal_pending(current)) {
3759                         mutex_unlock(&dev->struct_mutex);
3760                         schedule();
3761                         mutex_lock(&dev->struct_mutex);
3762                         continue;
3763                 }
3764                 ret = -ERESTARTSYS;
3765                 break;
3766         }
3767         finish_wait(&dev_priv->pending_flip_queue, &wait);
3768
3769         return ret;
3770 }
3771
3772 int
3773 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3774                        struct drm_file *file_priv,
3775                        struct drm_i915_gem_execbuffer2 *args,
3776                        struct drm_i915_gem_exec_object2 *exec_list)
3777 {
3778         drm_i915_private_t *dev_priv = dev->dev_private;
3779         struct drm_gem_object **object_list = NULL;
3780         struct drm_gem_object *batch_obj;
3781         struct drm_i915_gem_object *obj_priv;
3782         struct drm_clip_rect *cliprects = NULL;
3783         struct drm_i915_gem_relocation_entry *relocs = NULL;
3784         int ret = 0, ret2, i, pinned = 0;
3785         uint64_t exec_offset;
3786         uint32_t seqno, flush_domains, reloc_index;
3787         int pin_tries, flips;
3788
3789 #if WATCH_EXEC
3790         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3791                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3792 #endif
3793
3794         if (args->buffer_count < 1) {
3795                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3796                 return -EINVAL;
3797         }
3798         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3799         if (object_list == NULL) {
3800                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3801                           args->buffer_count);
3802                 ret = -ENOMEM;
3803                 goto pre_mutex_err;
3804         }
3805
3806         if (args->num_cliprects != 0) {
3807                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3808                                     GFP_KERNEL);
3809                 if (cliprects == NULL) {
3810                         ret = -ENOMEM;
3811                         goto pre_mutex_err;
3812                 }
3813
3814                 ret = copy_from_user(cliprects,
3815                                      (struct drm_clip_rect __user *)
3816                                      (uintptr_t) args->cliprects_ptr,
3817                                      sizeof(*cliprects) * args->num_cliprects);
3818                 if (ret != 0) {
3819                         DRM_ERROR("copy %d cliprects failed: %d\n",
3820                                   args->num_cliprects, ret);
3821                         goto pre_mutex_err;
3822                 }
3823         }
3824
3825         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3826                                             &relocs);
3827         if (ret != 0)
3828                 goto pre_mutex_err;
3829
3830         mutex_lock(&dev->struct_mutex);
3831
3832         i915_verify_inactive(dev, __FILE__, __LINE__);
3833
3834         if (atomic_read(&dev_priv->mm.wedged)) {
3835                 mutex_unlock(&dev->struct_mutex);
3836                 ret = -EIO;
3837                 goto pre_mutex_err;
3838         }
3839
3840         if (dev_priv->mm.suspended) {
3841                 mutex_unlock(&dev->struct_mutex);
3842                 ret = -EBUSY;
3843                 goto pre_mutex_err;
3844         }
3845
3846         /* Look up object handles */
3847         flips = 0;
3848         for (i = 0; i < args->buffer_count; i++) {
3849                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3850                                                        exec_list[i].handle);
3851                 if (object_list[i] == NULL) {
3852                         DRM_ERROR("Invalid object handle %d at index %d\n",
3853                                    exec_list[i].handle, i);
3854                         /* prevent error path from reading uninitialized data */
3855                         args->buffer_count = i + 1;
3856                         ret = -EBADF;
3857                         goto err;
3858                 }
3859
3860                 obj_priv = to_intel_bo(object_list[i]);
3861                 if (obj_priv->in_execbuffer) {
3862                         DRM_ERROR("Object %p appears more than once in object list\n",
3863                                    object_list[i]);
3864                         /* prevent error path from reading uninitialized data */
3865                         args->buffer_count = i + 1;
3866                         ret = -EBADF;
3867                         goto err;
3868                 }
3869                 obj_priv->in_execbuffer = true;
3870                 flips += atomic_read(&obj_priv->pending_flip);
3871         }
3872
3873         if (flips > 0) {
3874                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3875                                                      args->buffer_count);
3876                 if (ret)
3877                         goto err;
3878         }
3879
3880         /* Pin and relocate */
3881         for (pin_tries = 0; ; pin_tries++) {
3882                 ret = 0;
3883                 reloc_index = 0;
3884
3885                 for (i = 0; i < args->buffer_count; i++) {
3886                         object_list[i]->pending_read_domains = 0;
3887                         object_list[i]->pending_write_domain = 0;
3888                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3889                                                                file_priv,
3890                                                                &exec_list[i],
3891                                                                &relocs[reloc_index]);
3892                         if (ret)
3893                                 break;
3894                         pinned = i + 1;
3895                         reloc_index += exec_list[i].relocation_count;
3896                 }
3897                 /* success */
3898                 if (ret == 0)
3899                         break;
3900
3901                 /* error other than GTT full, or we've already tried again */
3902                 if (ret != -ENOSPC || pin_tries >= 1) {
3903                         if (ret != -ERESTARTSYS) {
3904                                 unsigned long long total_size = 0;
3905                                 for (i = 0; i < args->buffer_count; i++)
3906                                         total_size += object_list[i]->size;
3907                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes: %d\n",
3908                                           pinned+1, args->buffer_count,
3909                                           total_size, ret);
3910                                 DRM_ERROR("%d objects [%d pinned], "
3911                                           "%d object bytes [%d pinned], "
3912                                           "%d/%d gtt bytes\n",
3913                                           atomic_read(&dev->object_count),
3914                                           atomic_read(&dev->pin_count),
3915                                           atomic_read(&dev->object_memory),
3916                                           atomic_read(&dev->pin_memory),
3917                                           atomic_read(&dev->gtt_memory),
3918                                           dev->gtt_total);
3919                         }
3920                         goto err;
3921                 }
3922
3923                 /* unpin all of our buffers */
3924                 for (i = 0; i < pinned; i++)
3925                         i915_gem_object_unpin(object_list[i]);
3926                 pinned = 0;
3927
3928                 /* evict everyone we can from the aperture */
3929                 ret = i915_gem_evict_everything(dev);
3930                 if (ret && ret != -ENOSPC)
3931                         goto err;
3932         }
3933
3934         /* Set the pending read domains for the batch buffer to COMMAND */
3935         batch_obj = object_list[args->buffer_count-1];
3936         if (batch_obj->pending_write_domain) {
3937                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3938                 ret = -EINVAL;
3939                 goto err;
3940         }
3941         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3942
3943         /* Sanity check the batch buffer, prior to moving objects */
3944         exec_offset = exec_list[args->buffer_count - 1].offset;
3945         ret = i915_gem_check_execbuffer (args, exec_offset);
3946         if (ret != 0) {
3947                 DRM_ERROR("execbuf with invalid offset/length\n");
3948                 goto err;
3949         }
3950
3951         i915_verify_inactive(dev, __FILE__, __LINE__);
3952
3953         /* Zero the global flush/invalidate flags. These
3954          * will be modified as new domains are computed
3955          * for each object
3956          */
3957         dev->invalidate_domains = 0;
3958         dev->flush_domains = 0;
3959
3960         for (i = 0; i < args->buffer_count; i++) {
3961                 struct drm_gem_object *obj = object_list[i];
3962
3963                 /* Compute new gpu domains and update invalidate/flush */
3964                 i915_gem_object_set_to_gpu_domain(obj);
3965         }
3966
3967         i915_verify_inactive(dev, __FILE__, __LINE__);
3968
3969         if (dev->invalidate_domains | dev->flush_domains) {
3970 #if WATCH_EXEC
3971                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3972                           __func__,
3973                          dev->invalidate_domains,
3974                          dev->flush_domains);
3975 #endif
3976                 i915_gem_flush(dev,
3977                                dev->invalidate_domains,
3978                                dev->flush_domains);
3979                 if (dev->flush_domains & I915_GEM_GPU_DOMAINS)
3980                         (void)i915_add_request(dev, file_priv,
3981                                                dev->flush_domains);
3982         }
3983
3984         for (i = 0; i < args->buffer_count; i++) {
3985                 struct drm_gem_object *obj = object_list[i];
3986                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3987                 uint32_t old_write_domain = obj->write_domain;
3988
3989                 obj->write_domain = obj->pending_write_domain;
3990                 if (obj->write_domain)
3991                         list_move_tail(&obj_priv->gpu_write_list,
3992                                        &dev_priv->mm.gpu_write_list);
3993                 else
3994                         list_del_init(&obj_priv->gpu_write_list);
3995
3996                 trace_i915_gem_object_change_domain(obj,
3997                                                     obj->read_domains,
3998                                                     old_write_domain);
3999         }
4000
4001         i915_verify_inactive(dev, __FILE__, __LINE__);
4002
4003 #if WATCH_COHERENCY
4004         for (i = 0; i < args->buffer_count; i++) {
4005                 i915_gem_object_check_coherency(object_list[i],
4006                                                 exec_list[i].handle);
4007         }
4008 #endif
4009
4010 #if WATCH_EXEC
4011         i915_gem_dump_object(batch_obj,
4012                               args->batch_len,
4013                               __func__,
4014                               ~0);
4015 #endif
4016
4017         /* Exec the batchbuffer */
4018         ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
4019         if (ret) {
4020                 DRM_ERROR("dispatch failed %d\n", ret);
4021                 goto err;
4022         }
4023
4024         /*
4025          * Ensure that the commands in the batch buffer are
4026          * finished before the interrupt fires
4027          */
4028         flush_domains = i915_retire_commands(dev);
4029
4030         i915_verify_inactive(dev, __FILE__, __LINE__);
4031
4032         /*
4033          * Get a seqno representing the execution of the current buffer,
4034          * which we can wait on.  We would like to mitigate these interrupts,
4035          * likely by only creating seqnos occasionally (so that we have
4036          * *some* interrupts representing completion of buffers that we can
4037          * wait on when trying to clear up gtt space).
4038          */
4039         seqno = i915_add_request(dev, file_priv, flush_domains);
4040         BUG_ON(seqno == 0);
4041         for (i = 0; i < args->buffer_count; i++) {
4042                 struct drm_gem_object *obj = object_list[i];
4043
4044                 i915_gem_object_move_to_active(obj, seqno);
4045 #if WATCH_LRU
4046                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
4047 #endif
4048         }
4049 #if WATCH_LRU
4050         i915_dump_lru(dev, __func__);
4051 #endif
4052
4053         i915_verify_inactive(dev, __FILE__, __LINE__);
4054
4055 err:
4056         for (i = 0; i < pinned; i++)
4057                 i915_gem_object_unpin(object_list[i]);
4058
4059         for (i = 0; i < args->buffer_count; i++) {
4060                 if (object_list[i]) {
4061                         obj_priv = to_intel_bo(object_list[i]);
4062                         obj_priv->in_execbuffer = false;
4063                 }
4064                 drm_gem_object_unreference(object_list[i]);
4065         }
4066
4067         mutex_unlock(&dev->struct_mutex);
4068
4069 pre_mutex_err:
4070         /* Copy the updated relocations out regardless of current error
4071          * state.  Failure to update the relocs would mean that the next
4072          * time userland calls execbuf, it would do so with presumed offset
4073          * state that didn't match the actual object state.
4074          */
4075         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
4076                                            relocs);
4077         if (ret2 != 0) {
4078                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
4079
4080                 if (ret == 0)
4081                         ret = ret2;
4082         }
4083
4084         drm_free_large(object_list);
4085         kfree(cliprects);
4086
4087         return ret;
4088 }
4089
4090 /*
4091  * Legacy execbuffer just creates an exec2 list from the original exec object
4092  * list array and passes it to the real function.
4093  */
4094 int
4095 i915_gem_execbuffer(struct drm_device *dev, void *data,
4096                     struct drm_file *file_priv)
4097 {
4098         struct drm_i915_gem_execbuffer *args = data;
4099         struct drm_i915_gem_execbuffer2 exec2;
4100         struct drm_i915_gem_exec_object *exec_list = NULL;
4101         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4102         int ret, i;
4103
4104 #if WATCH_EXEC
4105         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4106                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4107 #endif
4108
4109         if (args->buffer_count < 1) {
4110                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4111                 return -EINVAL;
4112         }
4113
4114         /* Copy in the exec list from userland */
4115         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4116         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4117         if (exec_list == NULL || exec2_list == NULL) {
4118                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4119                           args->buffer_count);
4120                 drm_free_large(exec_list);
4121                 drm_free_large(exec2_list);
4122                 return -ENOMEM;
4123         }
4124         ret = copy_from_user(exec_list,
4125                              (struct drm_i915_relocation_entry __user *)
4126                              (uintptr_t) args->buffers_ptr,
4127                              sizeof(*exec_list) * args->buffer_count);
4128         if (ret != 0) {
4129                 DRM_ERROR("copy %d exec entries failed %d\n",
4130                           args->buffer_count, ret);
4131                 drm_free_large(exec_list);
4132                 drm_free_large(exec2_list);
4133                 return -EFAULT;
4134         }
4135
4136         for (i = 0; i < args->buffer_count; i++) {
4137                 exec2_list[i].handle = exec_list[i].handle;
4138                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4139                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4140                 exec2_list[i].alignment = exec_list[i].alignment;
4141                 exec2_list[i].offset = exec_list[i].offset;
4142                 if (!IS_I965G(dev))
4143                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4144                 else
4145                         exec2_list[i].flags = 0;
4146         }
4147
4148         exec2.buffers_ptr = args->buffers_ptr;
4149         exec2.buffer_count = args->buffer_count;
4150         exec2.batch_start_offset = args->batch_start_offset;
4151         exec2.batch_len = args->batch_len;
4152         exec2.DR1 = args->DR1;
4153         exec2.DR4 = args->DR4;
4154         exec2.num_cliprects = args->num_cliprects;
4155         exec2.cliprects_ptr = args->cliprects_ptr;
4156         exec2.flags = 0;
4157
4158         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4159         if (!ret) {
4160                 /* Copy the new buffer offsets back to the user's exec list. */
4161                 for (i = 0; i < args->buffer_count; i++)
4162                         exec_list[i].offset = exec2_list[i].offset;
4163                 /* ... and back out to userspace */
4164                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4165                                    (uintptr_t) args->buffers_ptr,
4166                                    exec_list,
4167                                    sizeof(*exec_list) * args->buffer_count);
4168                 if (ret) {
4169                         ret = -EFAULT;
4170                         DRM_ERROR("failed to copy %d exec entries "
4171                                   "back to user (%d)\n",
4172                                   args->buffer_count, ret);
4173                 }
4174         }
4175
4176         drm_free_large(exec_list);
4177         drm_free_large(exec2_list);
4178         return ret;
4179 }
4180
4181 int
4182 i915_gem_execbuffer2(struct drm_device *dev, void *data,
4183                      struct drm_file *file_priv)
4184 {
4185         struct drm_i915_gem_execbuffer2 *args = data;
4186         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4187         int ret;
4188
4189 #if WATCH_EXEC
4190         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4191                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4192 #endif
4193
4194         if (args->buffer_count < 1) {
4195                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4196                 return -EINVAL;
4197         }
4198
4199         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4200         if (exec2_list == NULL) {
4201                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4202                           args->buffer_count);
4203                 return -ENOMEM;
4204         }
4205         ret = copy_from_user(exec2_list,
4206                              (struct drm_i915_relocation_entry __user *)
4207                              (uintptr_t) args->buffers_ptr,
4208                              sizeof(*exec2_list) * args->buffer_count);
4209         if (ret != 0) {
4210                 DRM_ERROR("copy %d exec entries failed %d\n",
4211                           args->buffer_count, ret);
4212                 drm_free_large(exec2_list);
4213                 return -EFAULT;
4214         }
4215
4216         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4217         if (!ret) {
4218                 /* Copy the new buffer offsets back to the user's exec list. */
4219                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4220                                    (uintptr_t) args->buffers_ptr,
4221                                    exec2_list,
4222                                    sizeof(*exec2_list) * args->buffer_count);
4223                 if (ret) {
4224                         ret = -EFAULT;
4225                         DRM_ERROR("failed to copy %d exec entries "
4226                                   "back to user (%d)\n",
4227                                   args->buffer_count, ret);
4228                 }
4229         }
4230
4231         drm_free_large(exec2_list);
4232         return ret;
4233 }
4234
4235 int
4236 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4237 {
4238         struct drm_device *dev = obj->dev;
4239         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4240         int ret;
4241
4242         i915_verify_inactive(dev, __FILE__, __LINE__);
4243         if (obj_priv->gtt_space == NULL) {
4244                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4245                 if (ret)
4246                         return ret;
4247         }
4248
4249         obj_priv->pin_count++;
4250
4251         /* If the object is not active and not pending a flush,
4252          * remove it from the inactive list
4253          */
4254         if (obj_priv->pin_count == 1) {
4255                 atomic_inc(&dev->pin_count);
4256                 atomic_add(obj->size, &dev->pin_memory);
4257                 if (!obj_priv->active &&
4258                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
4259                     !list_empty(&obj_priv->list))
4260                         list_del_init(&obj_priv->list);
4261         }
4262         i915_verify_inactive(dev, __FILE__, __LINE__);
4263
4264         return 0;
4265 }
4266
4267 void
4268 i915_gem_object_unpin(struct drm_gem_object *obj)
4269 {
4270         struct drm_device *dev = obj->dev;
4271         drm_i915_private_t *dev_priv = dev->dev_private;
4272         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4273
4274         i915_verify_inactive(dev, __FILE__, __LINE__);
4275         obj_priv->pin_count--;
4276         BUG_ON(obj_priv->pin_count < 0);
4277         BUG_ON(obj_priv->gtt_space == NULL);
4278
4279         /* If the object is no longer pinned, and is
4280          * neither active nor being flushed, then stick it on
4281          * the inactive list
4282          */
4283         if (obj_priv->pin_count == 0) {
4284                 if (!obj_priv->active &&
4285                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4286                         list_move_tail(&obj_priv->list,
4287                                        &dev_priv->mm.inactive_list);
4288                 atomic_dec(&dev->pin_count);
4289                 atomic_sub(obj->size, &dev->pin_memory);
4290         }
4291         i915_verify_inactive(dev, __FILE__, __LINE__);
4292 }
4293
4294 int
4295 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4296                    struct drm_file *file_priv)
4297 {
4298         struct drm_i915_gem_pin *args = data;
4299         struct drm_gem_object *obj;
4300         struct drm_i915_gem_object *obj_priv;
4301         int ret;
4302
4303         mutex_lock(&dev->struct_mutex);
4304
4305         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4306         if (obj == NULL) {
4307                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4308                           args->handle);
4309                 mutex_unlock(&dev->struct_mutex);
4310                 return -EBADF;
4311         }
4312         obj_priv = to_intel_bo(obj);
4313
4314         if (obj_priv->madv != I915_MADV_WILLNEED) {
4315                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4316                 drm_gem_object_unreference(obj);
4317                 mutex_unlock(&dev->struct_mutex);
4318                 return -EINVAL;
4319         }
4320
4321         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4322                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4323                           args->handle);
4324                 drm_gem_object_unreference(obj);
4325                 mutex_unlock(&dev->struct_mutex);
4326                 return -EINVAL;
4327         }
4328
4329         obj_priv->user_pin_count++;
4330         obj_priv->pin_filp = file_priv;
4331         if (obj_priv->user_pin_count == 1) {
4332                 ret = i915_gem_object_pin(obj, args->alignment);
4333                 if (ret != 0) {
4334                         drm_gem_object_unreference(obj);
4335                         mutex_unlock(&dev->struct_mutex);
4336                         return ret;
4337                 }
4338         }
4339
4340         /* XXX - flush the CPU caches for pinned objects
4341          * as the X server doesn't manage domains yet
4342          */
4343         i915_gem_object_flush_cpu_write_domain(obj);
4344         args->offset = obj_priv->gtt_offset;
4345         drm_gem_object_unreference(obj);
4346         mutex_unlock(&dev->struct_mutex);
4347
4348         return 0;
4349 }
4350
4351 int
4352 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4353                      struct drm_file *file_priv)
4354 {
4355         struct drm_i915_gem_pin *args = data;
4356         struct drm_gem_object *obj;
4357         struct drm_i915_gem_object *obj_priv;
4358
4359         mutex_lock(&dev->struct_mutex);
4360
4361         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4362         if (obj == NULL) {
4363                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4364                           args->handle);
4365                 mutex_unlock(&dev->struct_mutex);
4366                 return -EBADF;
4367         }
4368
4369         obj_priv = to_intel_bo(obj);
4370         if (obj_priv->pin_filp != file_priv) {
4371                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4372                           args->handle);
4373                 drm_gem_object_unreference(obj);
4374                 mutex_unlock(&dev->struct_mutex);
4375                 return -EINVAL;
4376         }
4377         obj_priv->user_pin_count--;
4378         if (obj_priv->user_pin_count == 0) {
4379                 obj_priv->pin_filp = NULL;
4380                 i915_gem_object_unpin(obj);
4381         }
4382
4383         drm_gem_object_unreference(obj);
4384         mutex_unlock(&dev->struct_mutex);
4385         return 0;
4386 }
4387
4388 int
4389 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4390                     struct drm_file *file_priv)
4391 {
4392         struct drm_i915_gem_busy *args = data;
4393         struct drm_gem_object *obj;
4394         struct drm_i915_gem_object *obj_priv;
4395
4396         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4397         if (obj == NULL) {
4398                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4399                           args->handle);
4400                 return -EBADF;
4401         }
4402
4403         mutex_lock(&dev->struct_mutex);
4404         /* Update the active list for the hardware's current position.
4405          * Otherwise this only updates on a delayed timer or when irqs are
4406          * actually unmasked, and our working set ends up being larger than
4407          * required.
4408          */
4409         i915_gem_retire_requests(dev);
4410
4411         obj_priv = to_intel_bo(obj);
4412         /* Don't count being on the flushing list against the object being
4413          * done.  Otherwise, a buffer left on the flushing list but not getting
4414          * flushed (because nobody's flushing that domain) won't ever return
4415          * unbusy and get reused by libdrm's bo cache.  The other expected
4416          * consumer of this interface, OpenGL's occlusion queries, also specs
4417          * that the objects get unbusy "eventually" without any interference.
4418          */
4419         args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
4420
4421         drm_gem_object_unreference(obj);
4422         mutex_unlock(&dev->struct_mutex);
4423         return 0;
4424 }
4425
4426 int
4427 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4428                         struct drm_file *file_priv)
4429 {
4430     return i915_gem_ring_throttle(dev, file_priv);
4431 }
4432
4433 int
4434 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4435                        struct drm_file *file_priv)
4436 {
4437         struct drm_i915_gem_madvise *args = data;
4438         struct drm_gem_object *obj;
4439         struct drm_i915_gem_object *obj_priv;
4440
4441         switch (args->madv) {
4442         case I915_MADV_DONTNEED:
4443         case I915_MADV_WILLNEED:
4444             break;
4445         default:
4446             return -EINVAL;
4447         }
4448
4449         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4450         if (obj == NULL) {
4451                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4452                           args->handle);
4453                 return -EBADF;
4454         }
4455
4456         mutex_lock(&dev->struct_mutex);
4457         obj_priv = to_intel_bo(obj);
4458
4459         if (obj_priv->pin_count) {
4460                 drm_gem_object_unreference(obj);
4461                 mutex_unlock(&dev->struct_mutex);
4462
4463                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4464                 return -EINVAL;
4465         }
4466
4467         if (obj_priv->madv != __I915_MADV_PURGED)
4468                 obj_priv->madv = args->madv;
4469
4470         /* if the object is no longer bound, discard its backing storage */
4471         if (i915_gem_object_is_purgeable(obj_priv) &&
4472             obj_priv->gtt_space == NULL)
4473                 i915_gem_object_truncate(obj);
4474
4475         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4476
4477         drm_gem_object_unreference(obj);
4478         mutex_unlock(&dev->struct_mutex);
4479
4480         return 0;
4481 }
4482
4483 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4484                                               size_t size)
4485 {
4486         struct drm_i915_gem_object *obj;
4487
4488         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4489         if (obj == NULL)
4490                 return NULL;
4491
4492         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4493                 kfree(obj);
4494                 return NULL;
4495         }
4496
4497         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4498         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4499
4500         obj->agp_type = AGP_USER_MEMORY;
4501         obj->base.driver_private = NULL;
4502         obj->fence_reg = I915_FENCE_REG_NONE;
4503         INIT_LIST_HEAD(&obj->list);
4504         INIT_LIST_HEAD(&obj->gpu_write_list);
4505         obj->madv = I915_MADV_WILLNEED;
4506
4507         trace_i915_gem_object_create(&obj->base);
4508
4509         return &obj->base;
4510 }
4511
4512 int i915_gem_init_object(struct drm_gem_object *obj)
4513 {
4514         BUG();
4515
4516         return 0;
4517 }
4518
4519 void i915_gem_free_object(struct drm_gem_object *obj)
4520 {
4521         struct drm_device *dev = obj->dev;
4522         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4523
4524         trace_i915_gem_object_destroy(obj);
4525
4526         while (obj_priv->pin_count > 0)
4527                 i915_gem_object_unpin(obj);
4528
4529         if (obj_priv->phys_obj)
4530                 i915_gem_detach_phys_object(dev, obj);
4531
4532         i915_gem_object_unbind(obj);
4533
4534         if (obj_priv->mmap_offset)
4535                 i915_gem_free_mmap_offset(obj);
4536
4537         drm_gem_object_release(obj);
4538
4539         kfree(obj_priv->page_cpu_valid);
4540         kfree(obj_priv->bit_17);
4541         kfree(obj_priv);
4542 }
4543
4544 /** Unbinds all inactive objects. */
4545 static int
4546 i915_gem_evict_from_inactive_list(struct drm_device *dev)
4547 {
4548         drm_i915_private_t *dev_priv = dev->dev_private;
4549
4550         while (!list_empty(&dev_priv->mm.inactive_list)) {
4551                 struct drm_gem_object *obj;
4552                 int ret;
4553
4554                 obj = &list_first_entry(&dev_priv->mm.inactive_list,
4555                                         struct drm_i915_gem_object,
4556                                         list)->base;
4557
4558                 ret = i915_gem_object_unbind(obj);
4559                 if (ret != 0) {
4560                         DRM_ERROR("Error unbinding object: %d\n", ret);
4561                         return ret;
4562                 }
4563         }
4564
4565         return 0;
4566 }
4567
4568 int
4569 i915_gem_idle(struct drm_device *dev)
4570 {
4571         drm_i915_private_t *dev_priv = dev->dev_private;
4572         int ret;
4573
4574         mutex_lock(&dev->struct_mutex);
4575
4576         if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
4577                 mutex_unlock(&dev->struct_mutex);
4578                 return 0;
4579         }
4580
4581         ret = i915_gpu_idle(dev);
4582         if (ret) {
4583                 mutex_unlock(&dev->struct_mutex);
4584                 return ret;
4585         }
4586
4587         /* Under UMS, be paranoid and evict. */
4588         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4589                 ret = i915_gem_evict_from_inactive_list(dev);
4590                 if (ret) {
4591                         mutex_unlock(&dev->struct_mutex);
4592                         return ret;
4593                 }
4594         }
4595
4596         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4597          * We need to replace this with a semaphore, or something.
4598          * And not confound mm.suspended!
4599          */
4600         dev_priv->mm.suspended = 1;
4601         del_timer(&dev_priv->hangcheck_timer);
4602
4603         i915_kernel_lost_context(dev);
4604         i915_gem_cleanup_ringbuffer(dev);
4605
4606         mutex_unlock(&dev->struct_mutex);
4607
4608         /* Cancel the retire work handler, which should be idle now. */
4609         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4610
4611         return 0;
4612 }
4613
4614 /*
4615  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4616  * over cache flushing.
4617  */
4618 static int
4619 i915_gem_init_pipe_control(struct drm_device *dev)
4620 {
4621         drm_i915_private_t *dev_priv = dev->dev_private;
4622         struct drm_gem_object *obj;
4623         struct drm_i915_gem_object *obj_priv;
4624         int ret;
4625
4626         obj = i915_gem_alloc_object(dev, 4096);
4627         if (obj == NULL) {
4628                 DRM_ERROR("Failed to allocate seqno page\n");
4629                 ret = -ENOMEM;
4630                 goto err;
4631         }
4632         obj_priv = to_intel_bo(obj);
4633         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4634
4635         ret = i915_gem_object_pin(obj, 4096);
4636         if (ret)
4637                 goto err_unref;
4638
4639         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4640         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4641         if (dev_priv->seqno_page == NULL)
4642                 goto err_unpin;
4643
4644         dev_priv->seqno_obj = obj;
4645         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4646
4647         return 0;
4648
4649 err_unpin:
4650         i915_gem_object_unpin(obj);
4651 err_unref:
4652         drm_gem_object_unreference(obj);
4653 err:
4654         return ret;
4655 }
4656
4657 static int
4658 i915_gem_init_hws(struct drm_device *dev)
4659 {
4660         drm_i915_private_t *dev_priv = dev->dev_private;
4661         struct drm_gem_object *obj;
4662         struct drm_i915_gem_object *obj_priv;
4663         int ret;
4664
4665         /* If we need a physical address for the status page, it's already
4666          * initialized at driver load time.
4667          */
4668         if (!I915_NEED_GFX_HWS(dev))
4669                 return 0;
4670
4671         obj = i915_gem_alloc_object(dev, 4096);
4672         if (obj == NULL) {
4673                 DRM_ERROR("Failed to allocate status page\n");
4674                 ret = -ENOMEM;
4675                 goto err;
4676         }
4677         obj_priv = to_intel_bo(obj);
4678         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4679
4680         ret = i915_gem_object_pin(obj, 4096);
4681         if (ret != 0) {
4682                 drm_gem_object_unreference(obj);
4683                 goto err_unref;
4684         }
4685
4686         dev_priv->status_gfx_addr = obj_priv->gtt_offset;
4687
4688         dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
4689         if (dev_priv->hw_status_page == NULL) {
4690                 DRM_ERROR("Failed to map status page.\n");
4691                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4692                 ret = -EINVAL;
4693                 goto err_unpin;
4694         }
4695
4696         if (HAS_PIPE_CONTROL(dev)) {
4697                 ret = i915_gem_init_pipe_control(dev);
4698                 if (ret)
4699                         goto err_unpin;
4700         }
4701
4702         dev_priv->hws_obj = obj;
4703         memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
4704         if (IS_GEN6(dev)) {
4705                 I915_WRITE(HWS_PGA_GEN6, dev_priv->status_gfx_addr);
4706                 I915_READ(HWS_PGA_GEN6); /* posting read */
4707         } else {
4708                 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
4709                 I915_READ(HWS_PGA); /* posting read */
4710         }
4711         DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
4712
4713         return 0;
4714
4715 err_unpin:
4716         i915_gem_object_unpin(obj);
4717 err_unref:
4718         drm_gem_object_unreference(obj);
4719 err:
4720         return 0;
4721 }
4722
4723 static void
4724 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4725 {
4726         drm_i915_private_t *dev_priv = dev->dev_private;
4727         struct drm_gem_object *obj;
4728         struct drm_i915_gem_object *obj_priv;
4729
4730         obj = dev_priv->seqno_obj;
4731         obj_priv = to_intel_bo(obj);
4732         kunmap(obj_priv->pages[0]);
4733         i915_gem_object_unpin(obj);
4734         drm_gem_object_unreference(obj);
4735         dev_priv->seqno_obj = NULL;
4736
4737         dev_priv->seqno_page = NULL;
4738 }
4739
4740 static void
4741 i915_gem_cleanup_hws(struct drm_device *dev)
4742 {
4743         drm_i915_private_t *dev_priv = dev->dev_private;
4744         struct drm_gem_object *obj;
4745         struct drm_i915_gem_object *obj_priv;
4746
4747         if (dev_priv->hws_obj == NULL)
4748                 return;
4749
4750         obj = dev_priv->hws_obj;
4751         obj_priv = to_intel_bo(obj);
4752
4753         kunmap(obj_priv->pages[0]);
4754         i915_gem_object_unpin(obj);
4755         drm_gem_object_unreference(obj);
4756         dev_priv->hws_obj = NULL;
4757
4758         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4759         dev_priv->hw_status_page = NULL;
4760
4761         if (HAS_PIPE_CONTROL(dev))
4762                 i915_gem_cleanup_pipe_control(dev);
4763
4764         /* Write high address into HWS_PGA when disabling. */
4765         I915_WRITE(HWS_PGA, 0x1ffff000);
4766 }
4767
4768 int
4769 i915_gem_init_ringbuffer(struct drm_device *dev)
4770 {
4771         drm_i915_private_t *dev_priv = dev->dev_private;
4772         struct drm_gem_object *obj;
4773         struct drm_i915_gem_object *obj_priv;
4774         drm_i915_ring_buffer_t *ring = &dev_priv->ring;
4775         int ret;
4776         u32 head;
4777
4778         ret = i915_gem_init_hws(dev);
4779         if (ret != 0)
4780                 return ret;
4781
4782         obj = i915_gem_alloc_object(dev, 128 * 1024);
4783         if (obj == NULL) {
4784                 DRM_ERROR("Failed to allocate ringbuffer\n");
4785                 i915_gem_cleanup_hws(dev);
4786                 return -ENOMEM;
4787         }
4788         obj_priv = to_intel_bo(obj);
4789
4790         ret = i915_gem_object_pin(obj, 4096);
4791         if (ret != 0) {
4792                 drm_gem_object_unreference(obj);
4793                 i915_gem_cleanup_hws(dev);
4794                 return ret;
4795         }
4796
4797         /* Set up the kernel mapping for the ring. */
4798         ring->Size = obj->size;
4799
4800         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4801         ring->map.size = obj->size;
4802         ring->map.type = 0;
4803         ring->map.flags = 0;
4804         ring->map.mtrr = 0;
4805
4806         drm_core_ioremap_wc(&ring->map, dev);
4807         if (ring->map.handle == NULL) {
4808                 DRM_ERROR("Failed to map ringbuffer.\n");
4809                 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4810                 i915_gem_object_unpin(obj);
4811                 drm_gem_object_unreference(obj);
4812                 i915_gem_cleanup_hws(dev);
4813                 return -EINVAL;
4814         }
4815         ring->ring_obj = obj;
4816         ring->virtual_start = ring->map.handle;
4817
4818         /* Stop the ring if it's running. */
4819         I915_WRITE(PRB0_CTL, 0);
4820         I915_WRITE(PRB0_TAIL, 0);
4821         I915_WRITE(PRB0_HEAD, 0);
4822
4823         /* Initialize the ring. */
4824         I915_WRITE(PRB0_START, obj_priv->gtt_offset);
4825         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4826
4827         /* G45 ring initialization fails to reset head to zero */
4828         if (head != 0) {
4829                 DRM_ERROR("Ring head not reset to zero "
4830                           "ctl %08x head %08x tail %08x start %08x\n",
4831                           I915_READ(PRB0_CTL),
4832                           I915_READ(PRB0_HEAD),
4833                           I915_READ(PRB0_TAIL),
4834                           I915_READ(PRB0_START));
4835                 I915_WRITE(PRB0_HEAD, 0);
4836
4837                 DRM_ERROR("Ring head forced to zero "
4838                           "ctl %08x head %08x tail %08x start %08x\n",
4839                           I915_READ(PRB0_CTL),
4840                           I915_READ(PRB0_HEAD),
4841                           I915_READ(PRB0_TAIL),
4842                           I915_READ(PRB0_START));
4843         }
4844
4845         I915_WRITE(PRB0_CTL,
4846                    ((obj->size - 4096) & RING_NR_PAGES) |
4847                    RING_NO_REPORT |
4848                    RING_VALID);
4849
4850         head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4851
4852         /* If the head is still not zero, the ring is dead */
4853         if (head != 0) {
4854                 DRM_ERROR("Ring initialization failed "
4855                           "ctl %08x head %08x tail %08x start %08x\n",
4856                           I915_READ(PRB0_CTL),
4857                           I915_READ(PRB0_HEAD),
4858                           I915_READ(PRB0_TAIL),
4859                           I915_READ(PRB0_START));
4860                 return -EIO;
4861         }
4862
4863         /* Update our cache of the ring state */
4864         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4865                 i915_kernel_lost_context(dev);
4866         else {
4867                 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4868                 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4869                 ring->space = ring->head - (ring->tail + 8);
4870                 if (ring->space < 0)
4871                         ring->space += ring->Size;
4872         }
4873
4874         if (IS_I9XX(dev) && !IS_GEN3(dev)) {
4875                 I915_WRITE(MI_MODE,
4876                            (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
4877         }
4878
4879         return 0;
4880 }
4881
4882 void
4883 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4884 {
4885         drm_i915_private_t *dev_priv = dev->dev_private;
4886
4887         if (dev_priv->ring.ring_obj == NULL)
4888                 return;
4889
4890         drm_core_ioremapfree(&dev_priv->ring.map, dev);
4891
4892         i915_gem_object_unpin(dev_priv->ring.ring_obj);
4893         drm_gem_object_unreference(dev_priv->ring.ring_obj);
4894         dev_priv->ring.ring_obj = NULL;
4895         memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4896
4897         i915_gem_cleanup_hws(dev);
4898 }
4899
4900 int
4901 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4902                        struct drm_file *file_priv)
4903 {
4904         drm_i915_private_t *dev_priv = dev->dev_private;
4905         int ret;
4906
4907         if (drm_core_check_feature(dev, DRIVER_MODESET))
4908                 return 0;
4909
4910         if (atomic_read(&dev_priv->mm.wedged)) {
4911                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4912                 atomic_set(&dev_priv->mm.wedged, 0);
4913         }
4914
4915         mutex_lock(&dev->struct_mutex);
4916         dev_priv->mm.suspended = 0;
4917
4918         ret = i915_gem_init_ringbuffer(dev);
4919         if (ret != 0) {
4920                 mutex_unlock(&dev->struct_mutex);
4921                 return ret;
4922         }
4923
4924         spin_lock(&dev_priv->mm.active_list_lock);
4925         BUG_ON(!list_empty(&dev_priv->mm.active_list));
4926         spin_unlock(&dev_priv->mm.active_list_lock);
4927
4928         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4929         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4930         BUG_ON(!list_empty(&dev_priv->mm.request_list));
4931         mutex_unlock(&dev->struct_mutex);
4932
4933         drm_irq_install(dev);
4934
4935         return 0;
4936 }
4937
4938 int
4939 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4940                        struct drm_file *file_priv)
4941 {
4942         if (drm_core_check_feature(dev, DRIVER_MODESET))
4943                 return 0;
4944
4945         drm_irq_uninstall(dev);
4946         return i915_gem_idle(dev);
4947 }
4948
4949 void
4950 i915_gem_lastclose(struct drm_device *dev)
4951 {
4952         int ret;
4953
4954         if (drm_core_check_feature(dev, DRIVER_MODESET))
4955                 return;
4956
4957         ret = i915_gem_idle(dev);
4958         if (ret)
4959                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4960 }
4961
4962 void
4963 i915_gem_load(struct drm_device *dev)
4964 {
4965         int i;
4966         drm_i915_private_t *dev_priv = dev->dev_private;
4967
4968         spin_lock_init(&dev_priv->mm.active_list_lock);
4969         INIT_LIST_HEAD(&dev_priv->mm.active_list);
4970         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4971         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4972         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4973         INIT_LIST_HEAD(&dev_priv->mm.request_list);
4974         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4975         for (i = 0; i < 16; i++)
4976                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4977         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4978                           i915_gem_retire_work_handler);
4979         dev_priv->mm.next_gem_seqno = 1;
4980
4981         spin_lock(&shrink_list_lock);
4982         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4983         spin_unlock(&shrink_list_lock);
4984
4985         /* Old X drivers will take 0-2 for front, back, depth buffers */
4986         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4987                 dev_priv->fence_reg_start = 3;
4988
4989         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4990                 dev_priv->num_fence_regs = 16;
4991         else
4992                 dev_priv->num_fence_regs = 8;
4993
4994         /* Initialize fence registers to zero */
4995         if (IS_I965G(dev)) {
4996                 for (i = 0; i < 16; i++)
4997                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4998         } else {
4999                 for (i = 0; i < 8; i++)
5000                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
5001                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
5002                         for (i = 0; i < 8; i++)
5003                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
5004         }
5005         i915_gem_detect_bit_6_swizzle(dev);
5006         init_waitqueue_head(&dev_priv->pending_flip_queue);
5007 }
5008
5009 /*
5010  * Create a physically contiguous memory object for this object
5011  * e.g. for cursor + overlay regs
5012  */
5013 int i915_gem_init_phys_object(struct drm_device *dev,
5014                               int id, int size)
5015 {
5016         drm_i915_private_t *dev_priv = dev->dev_private;
5017         struct drm_i915_gem_phys_object *phys_obj;
5018         int ret;
5019
5020         if (dev_priv->mm.phys_objs[id - 1] || !size)
5021                 return 0;
5022
5023         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
5024         if (!phys_obj)
5025                 return -ENOMEM;
5026
5027         phys_obj->id = id;
5028
5029         phys_obj->handle = drm_pci_alloc(dev, size, 0);
5030         if (!phys_obj->handle) {
5031                 ret = -ENOMEM;
5032                 goto kfree_obj;
5033         }
5034 #ifdef CONFIG_X86
5035         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
5036 #endif
5037
5038         dev_priv->mm.phys_objs[id - 1] = phys_obj;
5039
5040         return 0;
5041 kfree_obj:
5042         kfree(phys_obj);
5043         return ret;
5044 }
5045
5046 void i915_gem_free_phys_object(struct drm_device *dev, int id)
5047 {
5048         drm_i915_private_t *dev_priv = dev->dev_private;
5049         struct drm_i915_gem_phys_object *phys_obj;
5050
5051         if (!dev_priv->mm.phys_objs[id - 1])
5052                 return;
5053
5054         phys_obj = dev_priv->mm.phys_objs[id - 1];
5055         if (phys_obj->cur_obj) {
5056                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
5057         }
5058
5059 #ifdef CONFIG_X86
5060         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
5061 #endif
5062         drm_pci_free(dev, phys_obj->handle);
5063         kfree(phys_obj);
5064         dev_priv->mm.phys_objs[id - 1] = NULL;
5065 }
5066
5067 void i915_gem_free_all_phys_object(struct drm_device *dev)
5068 {
5069         int i;
5070
5071         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
5072                 i915_gem_free_phys_object(dev, i);
5073 }
5074
5075 void i915_gem_detach_phys_object(struct drm_device *dev,
5076                                  struct drm_gem_object *obj)
5077 {
5078         struct drm_i915_gem_object *obj_priv;
5079         int i;
5080         int ret;
5081         int page_count;
5082
5083         obj_priv = to_intel_bo(obj);
5084         if (!obj_priv->phys_obj)
5085                 return;
5086
5087         ret = i915_gem_object_get_pages(obj, 0);
5088         if (ret)
5089                 goto out;
5090
5091         page_count = obj->size / PAGE_SIZE;
5092
5093         for (i = 0; i < page_count; i++) {
5094                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
5095                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
5096
5097                 memcpy(dst, src, PAGE_SIZE);
5098                 kunmap_atomic(dst, KM_USER0);
5099         }
5100         drm_clflush_pages(obj_priv->pages, page_count);
5101         drm_agp_chipset_flush(dev);
5102
5103         i915_gem_object_put_pages(obj);
5104 out:
5105         obj_priv->phys_obj->cur_obj = NULL;
5106         obj_priv->phys_obj = NULL;
5107 }
5108
5109 int
5110 i915_gem_attach_phys_object(struct drm_device *dev,
5111                             struct drm_gem_object *obj, int id)
5112 {
5113         drm_i915_private_t *dev_priv = dev->dev_private;
5114         struct drm_i915_gem_object *obj_priv;
5115         int ret = 0;
5116         int page_count;
5117         int i;
5118
5119         if (id > I915_MAX_PHYS_OBJECT)
5120                 return -EINVAL;
5121
5122         obj_priv = to_intel_bo(obj);
5123
5124         if (obj_priv->phys_obj) {
5125                 if (obj_priv->phys_obj->id == id)
5126                         return 0;
5127                 i915_gem_detach_phys_object(dev, obj);
5128         }
5129
5130
5131         /* create a new object */
5132         if (!dev_priv->mm.phys_objs[id - 1]) {
5133                 ret = i915_gem_init_phys_object(dev, id,
5134                                                 obj->size);
5135                 if (ret) {
5136                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
5137                         goto out;
5138                 }
5139         }
5140
5141         /* bind to the object */
5142         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
5143         obj_priv->phys_obj->cur_obj = obj;
5144
5145         ret = i915_gem_object_get_pages(obj, 0);
5146         if (ret) {
5147                 DRM_ERROR("failed to get page list\n");
5148                 goto out;
5149         }
5150
5151         page_count = obj->size / PAGE_SIZE;
5152
5153         for (i = 0; i < page_count; i++) {
5154                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
5155                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
5156
5157                 memcpy(dst, src, PAGE_SIZE);
5158                 kunmap_atomic(src, KM_USER0);
5159         }
5160
5161         i915_gem_object_put_pages(obj);
5162
5163         return 0;
5164 out:
5165         return ret;
5166 }
5167
5168 static int
5169 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
5170                      struct drm_i915_gem_pwrite *args,
5171                      struct drm_file *file_priv)
5172 {
5173         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
5174         void *obj_addr;
5175         int ret;
5176         char __user *user_data;
5177
5178         user_data = (char __user *) (uintptr_t) args->data_ptr;
5179         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
5180
5181         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
5182         ret = copy_from_user(obj_addr, user_data, args->size);
5183         if (ret)
5184                 return -EFAULT;
5185
5186         drm_agp_chipset_flush(dev);
5187         return 0;
5188 }
5189
5190 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
5191 {
5192         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
5193
5194         /* Clean up our request list when the client is going away, so that
5195          * later retire_requests won't dereference our soon-to-be-gone
5196          * file_priv.
5197          */
5198         mutex_lock(&dev->struct_mutex);
5199         while (!list_empty(&i915_file_priv->mm.request_list))
5200                 list_del_init(i915_file_priv->mm.request_list.next);
5201         mutex_unlock(&dev->struct_mutex);
5202 }
5203
5204 static int
5205 i915_gpu_is_active(struct drm_device *dev)
5206 {
5207         drm_i915_private_t *dev_priv = dev->dev_private;
5208         int lists_empty;
5209
5210         spin_lock(&dev_priv->mm.active_list_lock);
5211         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
5212                       list_empty(&dev_priv->mm.active_list);
5213         spin_unlock(&dev_priv->mm.active_list_lock);
5214
5215         return !lists_empty;
5216 }
5217
5218 static int
5219 i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
5220 {
5221         drm_i915_private_t *dev_priv, *next_dev;
5222         struct drm_i915_gem_object *obj_priv, *next_obj;
5223         int cnt = 0;
5224         int would_deadlock = 1;
5225
5226         /* "fast-path" to count number of available objects */
5227         if (nr_to_scan == 0) {
5228                 spin_lock(&shrink_list_lock);
5229                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5230                         struct drm_device *dev = dev_priv->dev;
5231
5232                         if (mutex_trylock(&dev->struct_mutex)) {
5233                                 list_for_each_entry(obj_priv,
5234                                                     &dev_priv->mm.inactive_list,
5235                                                     list)
5236                                         cnt++;
5237                                 mutex_unlock(&dev->struct_mutex);
5238                         }
5239                 }
5240                 spin_unlock(&shrink_list_lock);
5241
5242                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5243         }
5244
5245         spin_lock(&shrink_list_lock);
5246
5247 rescan:
5248         /* first scan for clean buffers */
5249         list_for_each_entry_safe(dev_priv, next_dev,
5250                                  &shrink_list, mm.shrink_list) {
5251                 struct drm_device *dev = dev_priv->dev;
5252
5253                 if (! mutex_trylock(&dev->struct_mutex))
5254                         continue;
5255
5256                 spin_unlock(&shrink_list_lock);
5257
5258                 i915_gem_retire_requests(dev);
5259
5260                 list_for_each_entry_safe(obj_priv, next_obj,
5261                                          &dev_priv->mm.inactive_list,
5262                                          list) {
5263                         if (i915_gem_object_is_purgeable(obj_priv)) {
5264                                 i915_gem_object_unbind(&obj_priv->base);
5265                                 if (--nr_to_scan <= 0)
5266                                         break;
5267                         }
5268                 }
5269
5270                 spin_lock(&shrink_list_lock);
5271                 mutex_unlock(&dev->struct_mutex);
5272
5273                 would_deadlock = 0;
5274
5275                 if (nr_to_scan <= 0)
5276                         break;
5277         }
5278
5279         /* second pass, evict/count anything still on the inactive list */
5280         list_for_each_entry_safe(dev_priv, next_dev,
5281                                  &shrink_list, mm.shrink_list) {
5282                 struct drm_device *dev = dev_priv->dev;
5283
5284                 if (! mutex_trylock(&dev->struct_mutex))
5285                         continue;
5286
5287                 spin_unlock(&shrink_list_lock);
5288
5289                 list_for_each_entry_safe(obj_priv, next_obj,
5290                                          &dev_priv->mm.inactive_list,
5291                                          list) {
5292                         if (nr_to_scan > 0) {
5293                                 i915_gem_object_unbind(&obj_priv->base);
5294                                 nr_to_scan--;
5295                         } else
5296                                 cnt++;
5297                 }
5298
5299                 spin_lock(&shrink_list_lock);
5300                 mutex_unlock(&dev->struct_mutex);
5301
5302                 would_deadlock = 0;
5303         }
5304
5305         if (nr_to_scan) {
5306                 int active = 0;
5307
5308                 /*
5309                  * We are desperate for pages, so as a last resort, wait
5310                  * for the GPU to finish and discard whatever we can.
5311                  * This has a dramatic impact to reduce the number of
5312                  * OOM-killer events whilst running the GPU aggressively.
5313                  */
5314                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5315                         struct drm_device *dev = dev_priv->dev;
5316
5317                         if (!mutex_trylock(&dev->struct_mutex))
5318                                 continue;
5319
5320                         spin_unlock(&shrink_list_lock);
5321
5322                         if (i915_gpu_is_active(dev)) {
5323                                 i915_gpu_idle(dev);
5324                                 active++;
5325                         }
5326
5327                         spin_lock(&shrink_list_lock);
5328                         mutex_unlock(&dev->struct_mutex);
5329                 }
5330
5331                 if (active)
5332                         goto rescan;
5333         }
5334
5335         spin_unlock(&shrink_list_lock);
5336
5337         if (would_deadlock)
5338                 return -1;
5339         else if (cnt > 0)
5340                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5341         else
5342                 return 0;
5343 }
5344
5345 static struct shrinker shrinker = {
5346         .shrink = i915_gem_shrink,
5347         .seeks = DEFAULT_SEEKS,
5348 };
5349
5350 __init void
5351 i915_gem_shrinker_init(void)
5352 {
5353     register_shrinker(&shrinker);
5354 }
5355
5356 __exit void
5357 i915_gem_shrinker_exit(void)
5358 {
5359     unregister_shrinker(&shrinker);
5360 }