2 * drivers/staging/omapdrm/omap_gem.c
4 * Copyright (C) 2011 Texas Instruments
5 * Author: Rob Clark <rob.clark@linaro.org>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published by
9 * the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program. If not, see <http://www.gnu.org/licenses/>.
21 #include <linux/spinlock.h>
22 #include <linux/shmem_fs.h>
26 /* remove these once drm core helpers are merged */
27 struct page ** _drm_gem_get_pages(struct drm_gem_object *obj, gfp_t gfpmask);
28 void _drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
29 bool dirty, bool accessed);
32 * GEM buffer object implementation.
35 #define to_omap_bo(x) container_of(x, struct omap_gem_object, base)
37 /* note: we use upper 8 bits of flags for driver-internal flags: */
38 #define OMAP_BO_DMA 0x01000000 /* actually is physically contiguous */
39 #define OMAP_BO_EXT_SYNC 0x02000000 /* externally allocated sync object */
40 #define OMAP_BO_EXT_MEM 0x04000000 /* externally allocated memory */
43 struct omap_gem_object {
44 struct drm_gem_object base;
49 * If buffer is allocated physically contiguous, the OMAP_BO_DMA flag
50 * is set and the paddr is valid.
52 * Note that OMAP_BO_SCANOUT is a hint from userspace that DMA capable
53 * buffer is requested, but doesn't mean that it is. Use the
54 * OMAP_BO_DMA flag to determine if the buffer has a DMA capable
60 * Array of backing pages, if allocated. Note that pages are never
61 * allocated for buffers originally allocated from contiguous memory
66 * Virtual address, if mapped.
71 * sync-object allocated on demand (if needed)
73 * Per-buffer sync-object for tracking pending and completed hw/dma
74 * read and write operations. The layout in memory is dictated by
75 * the SGX firmware, which uses this information to stall the command
76 * stream if a surface is not ready yet.
78 * Note that when buffer is used by SGX, the sync-object needs to be
79 * allocated from a special heap of sync-objects. This way many sync
80 * objects can be packed in a page, and not waste GPU virtual address
81 * space. Because of this we have to have a omap_gem_set_sync_object()
82 * API to allow replacement of the syncobj after it has (potentially)
83 * already been allocated. A bit ugly but I haven't thought of a
87 uint32_t write_pending;
88 uint32_t write_complete;
89 uint32_t read_pending;
90 uint32_t read_complete;
94 /* GEM objects can either be allocated from contiguous memory (in which
95 * case obj->filp==NULL), or w/ shmem backing (obj->filp!=NULL). But non
96 * contiguous buffers can be remapped in TILER/DMM if they need to be
97 * contiguous... but we don't do this all the time to reduce pressure
98 * on TILER/DMM space when we know at allocation time that the buffer
99 * will need to be scanned out.
101 static inline bool is_shmem(struct drm_gem_object *obj)
103 return obj->filp != NULL;
106 static int get_pages(struct drm_gem_object *obj, struct page ***pages);
108 static DEFINE_SPINLOCK(sync_lock);
110 /** ensure backing pages are allocated */
111 static int omap_gem_attach_pages(struct drm_gem_object *obj)
113 struct omap_gem_object *omap_obj = to_omap_bo(obj);
116 WARN_ON(omap_obj->pages);
118 /* TODO: __GFP_DMA32 .. but somehow GFP_HIGHMEM is coming from the
119 * mapping_gfp_mask(mapping) which conflicts w/ GFP_DMA32.. probably
120 * we actually want CMA memory for it all anyways..
122 pages = _drm_gem_get_pages(obj, GFP_KERNEL);
124 dev_err(obj->dev->dev, "could not get pages: %ld\n", PTR_ERR(pages));
125 return PTR_ERR(pages);
128 omap_obj->pages = pages;
132 /** release backing pages */
133 static void omap_gem_detach_pages(struct drm_gem_object *obj)
135 struct omap_gem_object *omap_obj = to_omap_bo(obj);
136 _drm_gem_put_pages(obj, omap_obj->pages, true, false);
137 omap_obj->pages = NULL;
140 /** get mmap offset */
141 uint64_t omap_gem_mmap_offset(struct drm_gem_object *obj)
143 if (!obj->map_list.map) {
144 /* Make it mmapable */
145 int ret = drm_gem_create_mmap_offset(obj);
147 dev_err(obj->dev->dev, "could not allocate mmap offset");
152 return (uint64_t)obj->map_list.hash.key << PAGE_SHIFT;
156 * omap_gem_fault - pagefault handler for GEM objects
157 * @vma: the VMA of the GEM object
160 * Invoked when a fault occurs on an mmap of a GEM managed area. GEM
161 * does most of the work for us including the actual map/unmap calls
162 * but we need to do the actual page work.
164 * The VMA was set up by GEM. In doing so it also ensured that the
165 * vma->vm_private_data points to the GEM object that is backing this
168 int omap_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
170 struct drm_gem_object *obj = vma->vm_private_data;
171 struct omap_gem_object *omap_obj = to_omap_bo(obj);
172 struct drm_device *dev = obj->dev;
178 /* Make sure we don't parallel update on a fault, nor move or remove
179 * something from beneath our feet
181 mutex_lock(&dev->struct_mutex);
183 /* if a shmem backed object, make sure we have pages attached now */
184 ret = get_pages(obj, &pages);
189 /* where should we do corresponding put_pages().. we are mapping
190 * the original page, rather than thru a GART, so we can't rely
191 * on eviction to trigger this. But munmap() or all mappings should
192 * probably trigger put_pages()?
195 /* We don't use vmf->pgoff since that has the fake offset: */
196 pgoff = ((unsigned long)vmf->virtual_address -
197 vma->vm_start) >> PAGE_SHIFT;
199 if (omap_obj->pages) {
200 pfn = page_to_pfn(omap_obj->pages[pgoff]);
202 BUG_ON(!(omap_obj->flags & OMAP_BO_DMA));
203 pfn = (omap_obj->paddr >> PAGE_SHIFT) + pgoff;
206 VERB("Inserting %p pfn %lx, pa %lx", vmf->virtual_address,
207 pfn, pfn << PAGE_SHIFT);
209 ret = vm_insert_mixed(vma, (unsigned long)vmf->virtual_address, pfn);
212 mutex_unlock(&dev->struct_mutex);
217 return VM_FAULT_NOPAGE;
221 return VM_FAULT_SIGBUS;
225 /** We override mainly to fix up some of the vm mapping flags.. */
226 int omap_gem_mmap(struct file *filp, struct vm_area_struct *vma)
228 struct omap_gem_object *omap_obj;
231 ret = drm_gem_mmap(filp, vma);
233 DBG("mmap failed: %d", ret);
237 /* after drm_gem_mmap(), it is safe to access the obj */
238 omap_obj = to_omap_bo(vma->vm_private_data);
240 vma->vm_flags &= ~VM_PFNMAP;
241 vma->vm_flags |= VM_MIXEDMAP;
243 if (omap_obj->flags & OMAP_BO_WC) {
244 vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
245 } else if (omap_obj->flags & OMAP_BO_UNCACHED) {
246 vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
248 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
255 * omap_gem_dumb_create - create a dumb buffer
256 * @drm_file: our client file
258 * @args: the requested arguments copied from userspace
260 * Allocate a buffer suitable for use for a frame buffer of the
261 * form described by user space. Give userspace a handle by which
264 int omap_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
265 struct drm_mode_create_dumb *args)
267 union omap_gem_size gsize;
269 /* in case someone tries to feed us a completely bogus stride: */
270 args->pitch = align_pitch(args->pitch, args->width, args->bpp);
271 args->size = PAGE_ALIGN(args->pitch * args->height);
273 gsize = (union omap_gem_size){
277 return omap_gem_new_handle(dev, file, gsize,
278 OMAP_BO_SCANOUT | OMAP_BO_WC, &args->handle);
282 * omap_gem_dumb_destroy - destroy a dumb buffer
284 * @dev: our DRM device
285 * @handle: the object handle
287 * Destroy a handle that was created via omap_gem_dumb_create.
289 int omap_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev,
292 /* No special work needed, drop the reference and see what falls out */
293 return drm_gem_handle_delete(file, handle);
297 * omap_gem_dumb_map - buffer mapping for dumb interface
298 * @file: our drm client file
300 * @handle: GEM handle to the object (from dumb_create)
302 * Do the necessary setup to allow the mapping of the frame buffer
303 * into user memory. We don't have to do much here at the moment.
305 int omap_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
306 uint32_t handle, uint64_t *offset)
308 struct drm_gem_object *obj;
311 mutex_lock(&dev->struct_mutex);
313 /* GEM does all our handle to object mapping */
314 obj = drm_gem_object_lookup(dev, file, handle);
320 *offset = omap_gem_mmap_offset(obj);
322 drm_gem_object_unreference_unlocked(obj);
325 mutex_unlock(&dev->struct_mutex);
329 /* Get physical address for DMA.. if 'remap' is true, and the buffer is not
330 * already contiguous, remap it to pin in physically contiguous memory.. (ie.
333 int omap_gem_get_paddr(struct drm_gem_object *obj,
334 dma_addr_t *paddr, bool remap)
336 struct omap_gem_object *omap_obj = to_omap_bo(obj);
340 /* TODO: remap to TILER */
344 *paddr = omap_obj->paddr;
349 /* Release physical address, when DMA is no longer being performed.. this
350 * could potentially unpin and unmap buffers from TILER
352 int omap_gem_put_paddr(struct drm_gem_object *obj)
354 /* do something here when remap to TILER is used.. */
358 /* acquire pages when needed (for example, for DMA where physically
359 * contiguous buffer is not required
361 static int get_pages(struct drm_gem_object *obj, struct page ***pages)
363 struct omap_gem_object *omap_obj = to_omap_bo(obj);
366 if (is_shmem(obj) && !omap_obj->pages) {
367 ret = omap_gem_attach_pages(obj);
369 dev_err(obj->dev->dev, "could not attach pages\n");
374 /* TODO: even phys-contig.. we should have a list of pages? */
375 *pages = omap_obj->pages;
380 int omap_gem_get_pages(struct drm_gem_object *obj, struct page ***pages)
383 mutex_lock(&obj->dev->struct_mutex);
384 ret = get_pages(obj, pages);
385 mutex_unlock(&obj->dev->struct_mutex);
389 /* release pages when DMA no longer being performed */
390 int omap_gem_put_pages(struct drm_gem_object *obj)
392 /* do something here if we dynamically attach/detach pages.. at
393 * least they would no longer need to be pinned if everyone has
394 * released the pages..
399 /* Get kernel virtual address for CPU access.. only buffers that are
400 * allocated contiguously have a kernel virtual address, so this more
401 * or less only exists for omap_fbdev
403 void *omap_gem_vaddr(struct drm_gem_object *obj)
405 struct omap_gem_object *omap_obj = to_omap_bo(obj);
406 return omap_obj->vaddr;
409 /* Buffer Synchronization:
412 struct omap_gem_sync_waiter {
413 struct list_head list;
414 struct omap_gem_object *omap_obj;
416 uint32_t read_target, write_target;
417 /* notify called w/ sync_lock held */
418 void (*notify)(void *arg);
422 /* list of omap_gem_sync_waiter.. the notify fxn gets called back when
423 * the read and/or write target count is achieved which can call a user
424 * callback (ex. to kick 3d and/or 2d), wakeup blocked task (prep for
427 static LIST_HEAD(waiters);
429 static inline bool is_waiting(struct omap_gem_sync_waiter *waiter)
431 struct omap_gem_object *omap_obj = waiter->omap_obj;
432 if ((waiter->op & OMAP_GEM_READ) &&
433 (omap_obj->sync->read_complete < waiter->read_target))
435 if ((waiter->op & OMAP_GEM_WRITE) &&
436 (omap_obj->sync->write_complete < waiter->write_target))
441 /* macro for sync debug.. */
443 #define SYNC(fmt, ...) do { if (SYNCDBG) \
444 printk(KERN_ERR "%s:%d: "fmt"\n", \
445 __func__, __LINE__, ##__VA_ARGS__); \
449 static void sync_op_update(void)
451 struct omap_gem_sync_waiter *waiter, *n;
452 list_for_each_entry_safe(waiter, n, &waiters, list) {
453 if (!is_waiting(waiter)) {
454 list_del(&waiter->list);
455 SYNC("notify: %p", waiter);
456 waiter->notify(waiter->arg);
462 static inline int sync_op(struct drm_gem_object *obj,
463 enum omap_gem_op op, bool start)
465 struct omap_gem_object *omap_obj = to_omap_bo(obj);
468 spin_lock(&sync_lock);
470 if (!omap_obj->sync) {
471 omap_obj->sync = kzalloc(sizeof(*omap_obj->sync), GFP_ATOMIC);
472 if (!omap_obj->sync) {
479 if (op & OMAP_GEM_READ)
480 omap_obj->sync->read_pending++;
481 if (op & OMAP_GEM_WRITE)
482 omap_obj->sync->write_pending++;
484 if (op & OMAP_GEM_READ)
485 omap_obj->sync->read_complete++;
486 if (op & OMAP_GEM_WRITE)
487 omap_obj->sync->write_complete++;
492 spin_unlock(&sync_lock);
497 /* it is a bit lame to handle updates in this sort of polling way, but
498 * in case of PVR, the GPU can directly update read/write complete
499 * values, and not really tell us which ones it updated.. this also
500 * means that sync_lock is not quite sufficient. So we'll need to
501 * do something a bit better when it comes time to add support for
504 void omap_gem_op_update(void)
506 spin_lock(&sync_lock);
508 spin_unlock(&sync_lock);
511 /* mark the start of read and/or write operation */
512 int omap_gem_op_start(struct drm_gem_object *obj, enum omap_gem_op op)
514 return sync_op(obj, op, true);
517 int omap_gem_op_finish(struct drm_gem_object *obj, enum omap_gem_op op)
519 return sync_op(obj, op, false);
522 static DECLARE_WAIT_QUEUE_HEAD(sync_event);
524 static void sync_notify(void *arg)
526 struct task_struct **waiter_task = arg;
528 wake_up_all(&sync_event);
531 int omap_gem_op_sync(struct drm_gem_object *obj, enum omap_gem_op op)
533 struct omap_gem_object *omap_obj = to_omap_bo(obj);
535 if (omap_obj->sync) {
536 struct task_struct *waiter_task = current;
537 struct omap_gem_sync_waiter *waiter =
538 kzalloc(sizeof(*waiter), GFP_KERNEL);
544 waiter->omap_obj = omap_obj;
546 waiter->read_target = omap_obj->sync->read_pending;
547 waiter->write_target = omap_obj->sync->write_pending;
548 waiter->notify = sync_notify;
549 waiter->arg = &waiter_task;
551 spin_lock(&sync_lock);
552 if (is_waiting(waiter)) {
553 SYNC("waited: %p", waiter);
554 list_add_tail(&waiter->list, &waiters);
555 spin_unlock(&sync_lock);
556 ret = wait_event_interruptible(sync_event,
557 (waiter_task == NULL));
558 spin_lock(&sync_lock);
560 SYNC("interrupted: %p", waiter);
561 /* we were interrupted */
562 list_del(&waiter->list);
565 /* freed in sync_op_update() */
569 spin_unlock(&sync_lock);
578 /* call fxn(arg), either synchronously or asynchronously if the op
579 * is currently blocked.. fxn() can be called from any context
581 * (TODO for now fxn is called back from whichever context calls
582 * omap_gem_op_update().. but this could be better defined later
585 * TODO more code in common w/ _sync()..
587 int omap_gem_op_async(struct drm_gem_object *obj, enum omap_gem_op op,
588 void (*fxn)(void *arg), void *arg)
590 struct omap_gem_object *omap_obj = to_omap_bo(obj);
591 if (omap_obj->sync) {
592 struct omap_gem_sync_waiter *waiter =
593 kzalloc(sizeof(*waiter), GFP_ATOMIC);
599 waiter->omap_obj = omap_obj;
601 waiter->read_target = omap_obj->sync->read_pending;
602 waiter->write_target = omap_obj->sync->write_pending;
603 waiter->notify = fxn;
606 spin_lock(&sync_lock);
607 if (is_waiting(waiter)) {
608 SYNC("waited: %p", waiter);
609 list_add_tail(&waiter->list, &waiters);
610 spin_unlock(&sync_lock);
614 spin_unlock(&sync_lock);
623 /* special API so PVR can update the buffer to use a sync-object allocated
624 * from it's sync-obj heap. Only used for a newly allocated (from PVR's
625 * perspective) sync-object, so we overwrite the new syncobj w/ values
626 * from the already allocated syncobj (if there is one)
628 int omap_gem_set_sync_object(struct drm_gem_object *obj, void *syncobj)
630 struct omap_gem_object *omap_obj = to_omap_bo(obj);
633 spin_lock(&sync_lock);
635 if ((omap_obj->flags & OMAP_BO_EXT_SYNC) && !syncobj) {
636 /* clearing a previously set syncobj */
637 syncobj = kzalloc(sizeof(*omap_obj->sync), GFP_ATOMIC);
642 memcpy(syncobj, omap_obj->sync, sizeof(*omap_obj->sync));
643 omap_obj->flags &= ~OMAP_BO_EXT_SYNC;
644 omap_obj->sync = syncobj;
645 } else if (syncobj && !(omap_obj->flags & OMAP_BO_EXT_SYNC)) {
646 /* replacing an existing syncobj */
647 if (omap_obj->sync) {
648 memcpy(syncobj, omap_obj->sync, sizeof(*omap_obj->sync));
649 kfree(omap_obj->sync);
651 omap_obj->flags |= OMAP_BO_EXT_SYNC;
652 omap_obj->sync = syncobj;
656 spin_unlock(&sync_lock);
660 int omap_gem_init_object(struct drm_gem_object *obj)
662 return -EINVAL; /* unused */
665 /* don't call directly.. called from GEM core when it is time to actually
668 void omap_gem_free_object(struct drm_gem_object *obj)
670 struct drm_device *dev = obj->dev;
671 struct omap_gem_object *omap_obj = to_omap_bo(obj);
673 if (obj->map_list.map) {
674 drm_gem_free_mmap_offset(obj);
677 /* don't free externally allocated backing memory */
678 if (!(omap_obj->flags & OMAP_BO_EXT_MEM)) {
679 if (omap_obj->pages) {
680 omap_gem_detach_pages(obj);
682 if (!is_shmem(obj)) {
683 dma_free_writecombine(dev->dev, obj->size,
684 omap_obj->vaddr, omap_obj->paddr);
688 /* don't free externally allocated syncobj */
689 if (!(omap_obj->flags & OMAP_BO_EXT_SYNC)) {
690 kfree(omap_obj->sync);
693 drm_gem_object_release(obj);
698 /* convenience method to construct a GEM buffer object, and userspace handle */
699 int omap_gem_new_handle(struct drm_device *dev, struct drm_file *file,
700 union omap_gem_size gsize, uint32_t flags, uint32_t *handle)
702 struct drm_gem_object *obj;
705 obj = omap_gem_new(dev, gsize, flags);
709 ret = drm_gem_handle_create(file, obj, handle);
711 drm_gem_object_release(obj);
712 kfree(obj); /* TODO isn't there a dtor to call? just copying i915 */
716 /* drop reference from allocate - handle holds it now */
717 drm_gem_object_unreference_unlocked(obj);
722 /* GEM buffer object constructor */
723 struct drm_gem_object *omap_gem_new(struct drm_device *dev,
724 union omap_gem_size gsize, uint32_t flags)
726 struct omap_gem_object *omap_obj;
727 struct drm_gem_object *obj = NULL;
731 if (flags & OMAP_BO_TILED) {
732 /* TODO: not implemented yet */
736 size = PAGE_ALIGN(gsize.bytes);
738 omap_obj = kzalloc(sizeof(*omap_obj), GFP_KERNEL);
740 dev_err(dev->dev, "could not allocate GEM object\n");
744 obj = &omap_obj->base;
746 if (flags & OMAP_BO_SCANOUT) {
747 /* attempt to allocate contiguous memory */
748 omap_obj->vaddr = dma_alloc_writecombine(dev->dev, size,
749 &omap_obj->paddr, GFP_KERNEL);
750 if (omap_obj->vaddr) {
751 flags |= OMAP_BO_DMA;
755 omap_obj->flags = flags;
757 if (flags & (OMAP_BO_DMA|OMAP_BO_EXT_MEM)) {
758 ret = drm_gem_private_object_init(dev, obj, size);
760 ret = drm_gem_object_init(dev, obj, size);
771 omap_gem_free_object(obj);