#include <linux/swap.h>
#include <linux/pci.h>
-static void i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
+static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
-static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
- bool write);
-static int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
- uint64_t offset,
- uint64_t size);
+static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
+ bool write);
+static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
+ uint64_t offset,
+ uint64_t size);
static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
-static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
- unsigned alignment,
- bool map_and_fenceable);
+static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
+ unsigned alignment,
+ bool map_and_fenceable);
static void i915_gem_clear_fence_reg(struct drm_device *dev,
struct drm_i915_fence_reg *reg);
static int i915_gem_phys_pwrite(struct drm_device *dev,
{
drm_i915_private_t *dev_priv;
struct drm_device *dev;
+ bool idle;
+ int i;
dev_priv = container_of(work, drm_i915_private_t,
mm.retire_work.work);
i915_gem_retire_requests(dev);
- if (!dev_priv->mm.suspended &&
- (!list_empty(&dev_priv->ring[RCS].request_list) ||
- !list_empty(&dev_priv->ring[VCS].request_list) ||
- !list_empty(&dev_priv->ring[BCS].request_list)))
+ /* Send a periodic flush down the ring so we don't hold onto GEM
+ * objects indefinitely.
+ */
+ idle = true;
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ struct intel_ring_buffer *ring = &dev_priv->ring[i];
+
+ if (!list_empty(&ring->gpu_write_list)) {
+ struct drm_i915_gem_request *request;
+ int ret;
+
+ ret = i915_gem_flush_ring(dev, ring, 0,
+ I915_GEM_GPU_DOMAINS);
+ request = kzalloc(sizeof(*request), GFP_KERNEL);
+ if (ret || request == NULL ||
+ i915_add_request(dev, NULL, request, ring))
+ kfree(request);
+ }
+
+ idle &= list_empty(&ring->request_list);
+ }
+
+ if (!dev_priv->mm.suspended && !idle)
queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
+
mutex_unlock(&dev->struct_mutex);
}
return ret;
}
-void
+int
i915_gem_flush_ring(struct drm_device *dev,
struct intel_ring_buffer *ring,
uint32_t invalidate_domains,
uint32_t flush_domains)
{
- ring->flush(ring, invalidate_domains, flush_domains);
+ int ret;
+
+ ret = ring->flush(ring, invalidate_domains, flush_domains);
+ if (ret)
+ return ret;
+
i915_gem_process_flushing_list(dev, flush_domains, ring);
+ return 0;
}
static int i915_ring_idle(struct drm_device *dev,
struct intel_ring_buffer *ring)
{
+ int ret;
+
if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
return 0;
- if (!list_empty(&ring->gpu_write_list))
- i915_gem_flush_ring(dev, ring,
+ if (!list_empty(&ring->gpu_write_list)) {
+ ret = i915_gem_flush_ring(dev, ring,
I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+ if (ret)
+ return ret;
+ }
+
return i915_wait_request(dev,
i915_gem_next_request_seqno(dev, ring),
ring);
int ret;
if (obj->fenced_gpu_access) {
- if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
- i915_gem_flush_ring(obj->base.dev,
- obj->last_fenced_ring,
- 0, obj->base.write_domain);
+ if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+ ret = i915_gem_flush_ring(obj->base.dev,
+ obj->last_fenced_ring,
+ 0, obj->base.write_domain);
+ if (ret)
+ return ret;
+ }
obj->fenced_gpu_access = false;
}
obj->last_fenced_ring = NULL;
}
+ /* Ensure that all CPU reads are completed before installing a fence
+ * and all writes before removing the fence.
+ */
+ if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
+ mb();
+
return 0;
}
return ret;
} else if (obj->tiling_changed) {
if (obj->fenced_gpu_access) {
- if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
- i915_gem_flush_ring(obj->base.dev, obj->ring,
- 0, obj->base.write_domain);
+ if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+ ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
+ 0, obj->base.write_domain);
+ if (ret)
+ return ret;
+ }
obj->fenced_gpu_access = false;
}
obj->gtt_space = NULL;
if (ret == -ENOMEM) {
- /* first try to clear up some space from the GTT */
- ret = i915_gem_evict_something(dev, size,
- alignment,
- map_and_fenceable);
+ /* first try to reclaim some memory by clearing the GTT */
+ ret = i915_gem_evict_everything(dev, false);
if (ret) {
/* now try to shrink everyone else */
if (gfpmask) {
goto search_free;
}
- return ret;
+ return -ENOMEM;
}
goto search_free;
drm_mm_put_block(obj->gtt_space);
obj->gtt_space = NULL;
- ret = i915_gem_evict_something(dev, size,
- alignment, map_and_fenceable);
- if (ret)
+ if (i915_gem_evict_everything(dev, false))
return ret;
goto search_free;
}
/** Flushes any GPU write domain for the object if it's dirty. */
-static void
+static int
i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
{
struct drm_device *dev = obj->base.dev;
if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
- return;
+ return 0;
/* Queue the GPU write cache flushing we need. */
- i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
- BUG_ON(obj->base.write_domain);
+ return i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain);
}
/** Flushes the GTT write domain for the object if it's dirty. */
if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
return;
- /* No actual flushing is required for the GTT write domain. Writes
+ /* No actual flushing is required for the GTT write domain. Writes
* to it immediately go to main memory as far as we know, so there's
* no chipset flush. It also doesn't land in render cache.
+ *
+ * However, we do have to enforce the order so that all writes through
+ * the GTT land before any writes to the device, such as updates to
+ * the GATT itself.
*/
+ wmb();
+
i915_gem_release_mmap(obj);
old_write_domain = obj->base.write_domain;
if (obj->gtt_space == NULL)
return -EINVAL;
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
+ if (ret)
+ return ret;
+
if (obj->pending_gpu_write || write) {
ret = i915_gem_object_wait_rendering(obj, true);
if (ret)
if (obj->gtt_space == NULL)
return -EINVAL;
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
+ if (ret)
+ return ret;
+
/* Currently, we are always called from an non-interruptible context. */
if (pipelined != obj->ring) {
i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj,
bool interruptible)
{
+ int ret;
+
if (!obj->active)
return 0;
- if (obj->base.write_domain & I915_GEM_GPU_DOMAINS)
- i915_gem_flush_ring(obj->base.dev, obj->ring,
- 0, obj->base.write_domain);
+ if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
+ ret = i915_gem_flush_ring(obj->base.dev, obj->ring,
+ 0, obj->base.write_domain);
+ if (ret)
+ return ret;
+ }
return i915_gem_object_wait_rendering(obj, interruptible);
}
uint32_t old_write_domain, old_read_domains;
int ret;
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
+ if (ret)
+ return ret;
+
ret = i915_gem_object_wait_rendering(obj, true);
if (ret)
return ret;
if (offset == 0 && size == obj->base.size)
return i915_gem_object_set_to_cpu_domain(obj, 0);
- i915_gem_object_flush_gpu_write_domain(obj);
+ ret = i915_gem_object_flush_gpu_write_domain(obj);
+ if (ret)
+ return ret;
+
ret = i915_gem_object_wait_rendering(obj, true);
if (ret)
return ret;
* flush earlier is beneficial.
*/
if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
- i915_gem_flush_ring(dev, obj->ring,
- 0, obj->base.write_domain);
+ ret = i915_gem_flush_ring(dev, obj->ring,
+ 0, obj->base.write_domain);
} else if (obj->ring->outstanding_lazy_request ==
obj->last_rendering_seqno) {
struct drm_i915_gem_request *request;
git.openpandora.org / pandora-kernel.git / blobdiff