1 /**************************************************************************
3 * Copyright © 2009 VMware, Inc., Palo Alto, CA., USA
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "vmwgfx_drv.h"
29 #include "vmwgfx_reg.h"
30 #include "ttm/ttm_bo_api.h"
31 #include "ttm/ttm_placement.h"
33 static int vmw_cmd_invalid(struct vmw_private *dev_priv,
34 struct vmw_sw_context *sw_context,
35 SVGA3dCmdHeader *header)
37 return capable(CAP_SYS_ADMIN) ? : -EINVAL;
40 static int vmw_cmd_ok(struct vmw_private *dev_priv,
41 struct vmw_sw_context *sw_context,
42 SVGA3dCmdHeader *header)
47 static void vmw_resource_to_validate_list(struct vmw_sw_context *sw_context,
48 struct vmw_resource **p_res)
50 struct vmw_resource *res = *p_res;
52 if (list_empty(&res->validate_head)) {
53 list_add_tail(&res->validate_head, &sw_context->resource_list);
56 vmw_resource_unreference(p_res);
60 * vmw_bo_to_validate_list - add a bo to a validate list
62 * @sw_context: The software context used for this command submission batch.
63 * @bo: The buffer object to add.
64 * @fence_flags: Fence flags to be or'ed with any other fence flags for
65 * this buffer on this submission batch.
66 * @p_val_node: If non-NULL Will be updated with the validate node number
69 * Returns -EINVAL if the limit of number of buffer objects per command
70 * submission is reached.
72 static int vmw_bo_to_validate_list(struct vmw_sw_context *sw_context,
73 struct ttm_buffer_object *bo,
78 struct ttm_validate_buffer *val_buf;
80 val_node = vmw_dmabuf_validate_node(bo, sw_context->cur_val_buf);
82 if (unlikely(val_node >= VMWGFX_MAX_VALIDATIONS)) {
83 DRM_ERROR("Max number of DMA buffers per submission"
88 val_buf = &sw_context->val_bufs[val_node];
89 if (unlikely(val_node == sw_context->cur_val_buf)) {
90 val_buf->new_sync_obj_arg = NULL;
91 val_buf->bo = ttm_bo_reference(bo);
92 list_add_tail(&val_buf->head, &sw_context->validate_nodes);
93 ++sw_context->cur_val_buf;
96 val_buf->new_sync_obj_arg = (void *)
97 ((unsigned long) val_buf->new_sync_obj_arg | fence_flags);
98 sw_context->fence_flags |= fence_flags;
101 *p_val_node = val_node;
106 static int vmw_cmd_cid_check(struct vmw_private *dev_priv,
107 struct vmw_sw_context *sw_context,
108 SVGA3dCmdHeader *header)
110 struct vmw_resource *ctx;
113 SVGA3dCmdHeader header;
118 cmd = container_of(header, struct vmw_cid_cmd, header);
119 if (likely(sw_context->cid_valid && cmd->cid == sw_context->last_cid))
122 ret = vmw_context_check(dev_priv, sw_context->tfile, cmd->cid,
124 if (unlikely(ret != 0)) {
125 DRM_ERROR("Could not find or use context %u\n",
126 (unsigned) cmd->cid);
130 sw_context->last_cid = cmd->cid;
131 sw_context->cid_valid = true;
132 sw_context->cur_ctx = ctx;
133 vmw_resource_to_validate_list(sw_context, &ctx);
138 static int vmw_cmd_sid_check(struct vmw_private *dev_priv,
139 struct vmw_sw_context *sw_context,
142 struct vmw_surface *srf;
144 struct vmw_resource *res;
146 if (*sid == SVGA3D_INVALID_ID)
149 if (likely((sw_context->sid_valid &&
150 *sid == sw_context->last_sid))) {
151 *sid = sw_context->sid_translation;
155 ret = vmw_user_surface_lookup_handle(dev_priv,
158 if (unlikely(ret != 0)) {
159 DRM_ERROR("Could ot find or use surface 0x%08x "
162 (unsigned long) sid);
166 ret = vmw_surface_validate(dev_priv, srf);
167 if (unlikely(ret != 0)) {
168 if (ret != -ERESTARTSYS)
169 DRM_ERROR("Could not validate surface.\n");
170 vmw_surface_unreference(&srf);
174 sw_context->last_sid = *sid;
175 sw_context->sid_valid = true;
176 sw_context->sid_translation = srf->res.id;
177 *sid = sw_context->sid_translation;
180 vmw_resource_to_validate_list(sw_context, &res);
186 static int vmw_cmd_set_render_target_check(struct vmw_private *dev_priv,
187 struct vmw_sw_context *sw_context,
188 SVGA3dCmdHeader *header)
191 SVGA3dCmdHeader header;
192 SVGA3dCmdSetRenderTarget body;
196 ret = vmw_cmd_cid_check(dev_priv, sw_context, header);
197 if (unlikely(ret != 0))
200 cmd = container_of(header, struct vmw_sid_cmd, header);
201 ret = vmw_cmd_sid_check(dev_priv, sw_context, &cmd->body.target.sid);
205 static int vmw_cmd_surface_copy_check(struct vmw_private *dev_priv,
206 struct vmw_sw_context *sw_context,
207 SVGA3dCmdHeader *header)
210 SVGA3dCmdHeader header;
211 SVGA3dCmdSurfaceCopy body;
215 cmd = container_of(header, struct vmw_sid_cmd, header);
216 ret = vmw_cmd_sid_check(dev_priv, sw_context, &cmd->body.src.sid);
217 if (unlikely(ret != 0))
219 return vmw_cmd_sid_check(dev_priv, sw_context, &cmd->body.dest.sid);
222 static int vmw_cmd_stretch_blt_check(struct vmw_private *dev_priv,
223 struct vmw_sw_context *sw_context,
224 SVGA3dCmdHeader *header)
227 SVGA3dCmdHeader header;
228 SVGA3dCmdSurfaceStretchBlt body;
232 cmd = container_of(header, struct vmw_sid_cmd, header);
233 ret = vmw_cmd_sid_check(dev_priv, sw_context, &cmd->body.src.sid);
234 if (unlikely(ret != 0))
236 return vmw_cmd_sid_check(dev_priv, sw_context, &cmd->body.dest.sid);
239 static int vmw_cmd_blt_surf_screen_check(struct vmw_private *dev_priv,
240 struct vmw_sw_context *sw_context,
241 SVGA3dCmdHeader *header)
244 SVGA3dCmdHeader header;
245 SVGA3dCmdBlitSurfaceToScreen body;
248 cmd = container_of(header, struct vmw_sid_cmd, header);
250 if (unlikely(!sw_context->kernel)) {
251 DRM_ERROR("Kernel only SVGA3d command: %u.\n", cmd->header.id);
255 return vmw_cmd_sid_check(dev_priv, sw_context, &cmd->body.srcImage.sid);
258 static int vmw_cmd_present_check(struct vmw_private *dev_priv,
259 struct vmw_sw_context *sw_context,
260 SVGA3dCmdHeader *header)
263 SVGA3dCmdHeader header;
264 SVGA3dCmdPresent body;
268 cmd = container_of(header, struct vmw_sid_cmd, header);
270 if (unlikely(!sw_context->kernel)) {
271 DRM_ERROR("Kernel only SVGA3d command: %u.\n", cmd->header.id);
275 return vmw_cmd_sid_check(dev_priv, sw_context, &cmd->body.sid);
279 * vmw_query_bo_switch_prepare - Prepare to switch pinned buffer for queries.
281 * @dev_priv: The device private structure.
282 * @cid: The hardware context for the next query.
283 * @new_query_bo: The new buffer holding query results.
284 * @sw_context: The software context used for this command submission.
286 * This function checks whether @new_query_bo is suitable for holding
287 * query results, and if another buffer currently is pinned for query
288 * results. If so, the function prepares the state of @sw_context for
289 * switching pinned buffers after successful submission of the current
290 * command batch. It also checks whether we're using a new query context.
291 * In that case, it makes sure we emit a query barrier for the old
292 * context before the current query buffer is fenced.
294 static int vmw_query_bo_switch_prepare(struct vmw_private *dev_priv,
296 struct ttm_buffer_object *new_query_bo,
297 struct vmw_sw_context *sw_context)
300 bool add_cid = false;
303 if (unlikely(new_query_bo != sw_context->cur_query_bo)) {
305 if (unlikely(new_query_bo->num_pages > 4)) {
306 DRM_ERROR("Query buffer too large.\n");
310 if (unlikely(sw_context->cur_query_bo != NULL)) {
311 BUG_ON(!sw_context->query_cid_valid);
313 cid_to_add = sw_context->cur_query_cid;
314 ret = vmw_bo_to_validate_list(sw_context,
315 sw_context->cur_query_bo,
316 DRM_VMW_FENCE_FLAG_EXEC,
318 if (unlikely(ret != 0))
321 sw_context->cur_query_bo = new_query_bo;
323 ret = vmw_bo_to_validate_list(sw_context,
324 dev_priv->dummy_query_bo,
325 DRM_VMW_FENCE_FLAG_EXEC,
327 if (unlikely(ret != 0))
332 if (unlikely(cid != sw_context->cur_query_cid &&
333 sw_context->query_cid_valid)) {
335 cid_to_add = sw_context->cur_query_cid;
338 sw_context->cur_query_cid = cid;
339 sw_context->query_cid_valid = true;
342 struct vmw_resource *ctx = sw_context->cur_ctx;
344 if (list_empty(&ctx->query_head))
345 list_add_tail(&ctx->query_head,
346 &sw_context->query_list);
347 ret = vmw_bo_to_validate_list(sw_context,
348 dev_priv->dummy_query_bo,
349 DRM_VMW_FENCE_FLAG_EXEC,
351 if (unlikely(ret != 0))
359 * vmw_query_bo_switch_commit - Finalize switching pinned query buffer
361 * @dev_priv: The device private structure.
362 * @sw_context: The software context used for this command submission batch.
364 * This function will check if we're switching query buffers, and will then,
365 * if no other query waits are issued this command submission batch,
366 * issue a dummy occlusion query wait used as a query barrier. When the fence
367 * object following that query wait has signaled, we are sure that all
368 * preseding queries have finished, and the old query buffer can be unpinned.
369 * However, since both the new query buffer and the old one are fenced with
370 * that fence, we can do an asynchronus unpin now, and be sure that the
371 * old query buffer won't be moved until the fence has signaled.
373 * As mentioned above, both the new - and old query buffers need to be fenced
374 * using a sequence emitted *after* calling this function.
376 static void vmw_query_bo_switch_commit(struct vmw_private *dev_priv,
377 struct vmw_sw_context *sw_context)
380 struct vmw_resource *ctx, *next_ctx;
384 * The validate list should still hold references to all
388 list_for_each_entry_safe(ctx, next_ctx, &sw_context->query_list,
390 list_del_init(&ctx->query_head);
392 BUG_ON(list_empty(&ctx->validate_head));
394 ret = vmw_fifo_emit_dummy_query(dev_priv, ctx->id);
396 if (unlikely(ret != 0))
397 DRM_ERROR("Out of fifo space for dummy query.\n");
400 if (dev_priv->pinned_bo != sw_context->cur_query_bo) {
401 if (dev_priv->pinned_bo) {
402 vmw_bo_pin(dev_priv->pinned_bo, false);
403 ttm_bo_unref(&dev_priv->pinned_bo);
406 vmw_bo_pin(sw_context->cur_query_bo, true);
409 * We pin also the dummy_query_bo buffer so that we
410 * don't need to validate it when emitting
411 * dummy queries in context destroy paths.
414 vmw_bo_pin(dev_priv->dummy_query_bo, true);
415 dev_priv->dummy_query_bo_pinned = true;
417 dev_priv->query_cid = sw_context->cur_query_cid;
418 dev_priv->pinned_bo =
419 ttm_bo_reference(sw_context->cur_query_bo);
424 * vmw_query_switch_backoff - clear query barrier list
425 * @sw_context: The sw context used for this submission batch.
427 * This function is used as part of an error path, where a previously
428 * set up list of query barriers needs to be cleared.
431 static void vmw_query_switch_backoff(struct vmw_sw_context *sw_context)
433 struct list_head *list, *next;
435 list_for_each_safe(list, next, &sw_context->query_list) {
440 static int vmw_translate_guest_ptr(struct vmw_private *dev_priv,
441 struct vmw_sw_context *sw_context,
443 struct vmw_dma_buffer **vmw_bo_p)
445 struct vmw_dma_buffer *vmw_bo = NULL;
446 struct ttm_buffer_object *bo;
447 uint32_t handle = ptr->gmrId;
448 struct vmw_relocation *reloc;
451 ret = vmw_user_dmabuf_lookup(sw_context->tfile, handle, &vmw_bo);
452 if (unlikely(ret != 0)) {
453 DRM_ERROR("Could not find or use GMR region.\n");
458 if (unlikely(sw_context->cur_reloc >= VMWGFX_MAX_RELOCATIONS)) {
459 DRM_ERROR("Max number relocations per submission"
465 reloc = &sw_context->relocs[sw_context->cur_reloc++];
466 reloc->location = ptr;
468 ret = vmw_bo_to_validate_list(sw_context, bo, DRM_VMW_FENCE_FLAG_EXEC,
470 if (unlikely(ret != 0))
477 vmw_dmabuf_unreference(&vmw_bo);
482 static int vmw_cmd_end_query(struct vmw_private *dev_priv,
483 struct vmw_sw_context *sw_context,
484 SVGA3dCmdHeader *header)
486 struct vmw_dma_buffer *vmw_bo;
487 struct vmw_query_cmd {
488 SVGA3dCmdHeader header;
493 cmd = container_of(header, struct vmw_query_cmd, header);
494 ret = vmw_cmd_cid_check(dev_priv, sw_context, header);
495 if (unlikely(ret != 0))
498 ret = vmw_translate_guest_ptr(dev_priv, sw_context,
501 if (unlikely(ret != 0))
504 ret = vmw_query_bo_switch_prepare(dev_priv, cmd->q.cid,
505 &vmw_bo->base, sw_context);
507 vmw_dmabuf_unreference(&vmw_bo);
511 static int vmw_cmd_wait_query(struct vmw_private *dev_priv,
512 struct vmw_sw_context *sw_context,
513 SVGA3dCmdHeader *header)
515 struct vmw_dma_buffer *vmw_bo;
516 struct vmw_query_cmd {
517 SVGA3dCmdHeader header;
518 SVGA3dCmdWaitForQuery q;
521 struct vmw_resource *ctx;
523 cmd = container_of(header, struct vmw_query_cmd, header);
524 ret = vmw_cmd_cid_check(dev_priv, sw_context, header);
525 if (unlikely(ret != 0))
528 ret = vmw_translate_guest_ptr(dev_priv, sw_context,
531 if (unlikely(ret != 0))
534 vmw_dmabuf_unreference(&vmw_bo);
537 * This wait will act as a barrier for previous waits for this
541 ctx = sw_context->cur_ctx;
542 if (!list_empty(&ctx->query_head))
543 list_del_init(&ctx->query_head);
548 static int vmw_cmd_dma(struct vmw_private *dev_priv,
549 struct vmw_sw_context *sw_context,
550 SVGA3dCmdHeader *header)
552 struct vmw_dma_buffer *vmw_bo = NULL;
553 struct ttm_buffer_object *bo;
554 struct vmw_surface *srf = NULL;
556 SVGA3dCmdHeader header;
557 SVGA3dCmdSurfaceDMA dma;
560 struct vmw_resource *res;
561 SVGA3dCmdSurfaceDMASuffix *suffix;
564 cmd = container_of(header, struct vmw_dma_cmd, header);
565 suffix = (SVGA3dCmdSurfaceDMASuffix *)((unsigned long) &cmd->dma +
566 header->size - sizeof(*suffix));
568 /* Make sure device and verifier stays in sync. */
569 if (unlikely(suffix->suffixSize != sizeof(*suffix))) {
570 DRM_ERROR("Invalid DMA suffix size.\n");
574 ret = vmw_translate_guest_ptr(dev_priv, sw_context,
577 if (unlikely(ret != 0))
580 /* Make sure DMA doesn't cross BO boundaries. */
581 bo_size = vmw_bo->base.num_pages * PAGE_SIZE;
582 if (unlikely(cmd->dma.guest.ptr.offset > bo_size)) {
583 DRM_ERROR("Invalid DMA offset.\n");
587 bo_size -= cmd->dma.guest.ptr.offset;
588 if (unlikely(suffix->maximumOffset > bo_size))
589 suffix->maximumOffset = bo_size;
592 ret = vmw_user_surface_lookup_handle(dev_priv, sw_context->tfile,
593 cmd->dma.host.sid, &srf);
595 DRM_ERROR("could not find surface\n");
599 ret = vmw_surface_validate(dev_priv, srf);
600 if (unlikely(ret != 0)) {
601 if (ret != -ERESTARTSYS)
602 DRM_ERROR("Culd not validate surface.\n");
603 goto out_no_validate;
607 * Patch command stream with device SID.
609 cmd->dma.host.sid = srf->res.id;
610 vmw_kms_cursor_snoop(srf, sw_context->tfile, bo, header);
612 vmw_dmabuf_unreference(&vmw_bo);
615 vmw_resource_to_validate_list(sw_context, &res);
620 vmw_surface_unreference(&srf);
622 vmw_dmabuf_unreference(&vmw_bo);
626 static int vmw_cmd_draw(struct vmw_private *dev_priv,
627 struct vmw_sw_context *sw_context,
628 SVGA3dCmdHeader *header)
630 struct vmw_draw_cmd {
631 SVGA3dCmdHeader header;
632 SVGA3dCmdDrawPrimitives body;
634 SVGA3dVertexDecl *decl = (SVGA3dVertexDecl *)(
635 (unsigned long)header + sizeof(*cmd));
636 SVGA3dPrimitiveRange *range;
641 ret = vmw_cmd_cid_check(dev_priv, sw_context, header);
642 if (unlikely(ret != 0))
645 cmd = container_of(header, struct vmw_draw_cmd, header);
646 maxnum = (header->size - sizeof(cmd->body)) / sizeof(*decl);
648 if (unlikely(cmd->body.numVertexDecls > maxnum)) {
649 DRM_ERROR("Illegal number of vertex declarations.\n");
653 for (i = 0; i < cmd->body.numVertexDecls; ++i, ++decl) {
654 ret = vmw_cmd_sid_check(dev_priv, sw_context,
655 &decl->array.surfaceId);
656 if (unlikely(ret != 0))
660 maxnum = (header->size - sizeof(cmd->body) -
661 cmd->body.numVertexDecls * sizeof(*decl)) / sizeof(*range);
662 if (unlikely(cmd->body.numRanges > maxnum)) {
663 DRM_ERROR("Illegal number of index ranges.\n");
667 range = (SVGA3dPrimitiveRange *) decl;
668 for (i = 0; i < cmd->body.numRanges; ++i, ++range) {
669 ret = vmw_cmd_sid_check(dev_priv, sw_context,
670 &range->indexArray.surfaceId);
671 if (unlikely(ret != 0))
678 static int vmw_cmd_tex_state(struct vmw_private *dev_priv,
679 struct vmw_sw_context *sw_context,
680 SVGA3dCmdHeader *header)
682 struct vmw_tex_state_cmd {
683 SVGA3dCmdHeader header;
684 SVGA3dCmdSetTextureState state;
687 SVGA3dTextureState *last_state = (SVGA3dTextureState *)
688 ((unsigned long) header + header->size + sizeof(header));
689 SVGA3dTextureState *cur_state = (SVGA3dTextureState *)
690 ((unsigned long) header + sizeof(struct vmw_tex_state_cmd));
693 ret = vmw_cmd_cid_check(dev_priv, sw_context, header);
694 if (unlikely(ret != 0))
697 for (; cur_state < last_state; ++cur_state) {
698 if (likely(cur_state->name != SVGA3D_TS_BIND_TEXTURE))
701 ret = vmw_cmd_sid_check(dev_priv, sw_context,
703 if (unlikely(ret != 0))
710 static int vmw_cmd_check_define_gmrfb(struct vmw_private *dev_priv,
711 struct vmw_sw_context *sw_context,
714 struct vmw_dma_buffer *vmw_bo;
719 SVGAFifoCmdDefineGMRFB body;
722 ret = vmw_translate_guest_ptr(dev_priv, sw_context,
725 if (unlikely(ret != 0))
728 vmw_dmabuf_unreference(&vmw_bo);
733 static int vmw_cmd_check_not_3d(struct vmw_private *dev_priv,
734 struct vmw_sw_context *sw_context,
735 void *buf, uint32_t *size)
737 uint32_t size_remaining = *size;
740 cmd_id = le32_to_cpu(((uint32_t *)buf)[0]);
742 case SVGA_CMD_UPDATE:
743 *size = sizeof(uint32_t) + sizeof(SVGAFifoCmdUpdate);
745 case SVGA_CMD_DEFINE_GMRFB:
746 *size = sizeof(uint32_t) + sizeof(SVGAFifoCmdDefineGMRFB);
748 case SVGA_CMD_BLIT_GMRFB_TO_SCREEN:
749 *size = sizeof(uint32_t) + sizeof(SVGAFifoCmdBlitGMRFBToScreen);
751 case SVGA_CMD_BLIT_SCREEN_TO_GMRFB:
752 *size = sizeof(uint32_t) + sizeof(SVGAFifoCmdBlitGMRFBToScreen);
755 DRM_ERROR("Unsupported SVGA command: %u.\n", cmd_id);
759 if (*size > size_remaining) {
760 DRM_ERROR("Invalid SVGA command (size mismatch):"
765 if (unlikely(!sw_context->kernel)) {
766 DRM_ERROR("Kernel only SVGA command: %u.\n", cmd_id);
770 if (cmd_id == SVGA_CMD_DEFINE_GMRFB)
771 return vmw_cmd_check_define_gmrfb(dev_priv, sw_context, buf);
776 typedef int (*vmw_cmd_func) (struct vmw_private *,
777 struct vmw_sw_context *,
780 #define VMW_CMD_DEF(cmd, func) \
781 [cmd - SVGA_3D_CMD_BASE] = func
783 static vmw_cmd_func vmw_cmd_funcs[SVGA_3D_CMD_MAX] = {
784 VMW_CMD_DEF(SVGA_3D_CMD_SURFACE_DEFINE, &vmw_cmd_invalid),
785 VMW_CMD_DEF(SVGA_3D_CMD_SURFACE_DESTROY, &vmw_cmd_invalid),
786 VMW_CMD_DEF(SVGA_3D_CMD_SURFACE_COPY, &vmw_cmd_surface_copy_check),
787 VMW_CMD_DEF(SVGA_3D_CMD_SURFACE_STRETCHBLT, &vmw_cmd_stretch_blt_check),
788 VMW_CMD_DEF(SVGA_3D_CMD_SURFACE_DMA, &vmw_cmd_dma),
789 VMW_CMD_DEF(SVGA_3D_CMD_CONTEXT_DEFINE, &vmw_cmd_invalid),
790 VMW_CMD_DEF(SVGA_3D_CMD_CONTEXT_DESTROY, &vmw_cmd_invalid),
791 VMW_CMD_DEF(SVGA_3D_CMD_SETTRANSFORM, &vmw_cmd_cid_check),
792 VMW_CMD_DEF(SVGA_3D_CMD_SETZRANGE, &vmw_cmd_cid_check),
793 VMW_CMD_DEF(SVGA_3D_CMD_SETRENDERSTATE, &vmw_cmd_cid_check),
794 VMW_CMD_DEF(SVGA_3D_CMD_SETRENDERTARGET,
795 &vmw_cmd_set_render_target_check),
796 VMW_CMD_DEF(SVGA_3D_CMD_SETTEXTURESTATE, &vmw_cmd_tex_state),
797 VMW_CMD_DEF(SVGA_3D_CMD_SETMATERIAL, &vmw_cmd_cid_check),
798 VMW_CMD_DEF(SVGA_3D_CMD_SETLIGHTDATA, &vmw_cmd_cid_check),
799 VMW_CMD_DEF(SVGA_3D_CMD_SETLIGHTENABLED, &vmw_cmd_cid_check),
800 VMW_CMD_DEF(SVGA_3D_CMD_SETVIEWPORT, &vmw_cmd_cid_check),
801 VMW_CMD_DEF(SVGA_3D_CMD_SETCLIPPLANE, &vmw_cmd_cid_check),
802 VMW_CMD_DEF(SVGA_3D_CMD_CLEAR, &vmw_cmd_cid_check),
803 VMW_CMD_DEF(SVGA_3D_CMD_PRESENT, &vmw_cmd_present_check),
804 VMW_CMD_DEF(SVGA_3D_CMD_SHADER_DEFINE, &vmw_cmd_cid_check),
805 VMW_CMD_DEF(SVGA_3D_CMD_SHADER_DESTROY, &vmw_cmd_cid_check),
806 VMW_CMD_DEF(SVGA_3D_CMD_SET_SHADER, &vmw_cmd_cid_check),
807 VMW_CMD_DEF(SVGA_3D_CMD_SET_SHADER_CONST, &vmw_cmd_cid_check),
808 VMW_CMD_DEF(SVGA_3D_CMD_DRAW_PRIMITIVES, &vmw_cmd_draw),
809 VMW_CMD_DEF(SVGA_3D_CMD_SETSCISSORRECT, &vmw_cmd_cid_check),
810 VMW_CMD_DEF(SVGA_3D_CMD_BEGIN_QUERY, &vmw_cmd_cid_check),
811 VMW_CMD_DEF(SVGA_3D_CMD_END_QUERY, &vmw_cmd_end_query),
812 VMW_CMD_DEF(SVGA_3D_CMD_WAIT_FOR_QUERY, &vmw_cmd_wait_query),
813 VMW_CMD_DEF(SVGA_3D_CMD_PRESENT_READBACK, &vmw_cmd_ok),
814 VMW_CMD_DEF(SVGA_3D_CMD_BLIT_SURFACE_TO_SCREEN,
815 &vmw_cmd_blt_surf_screen_check)
818 static int vmw_cmd_check(struct vmw_private *dev_priv,
819 struct vmw_sw_context *sw_context,
820 void *buf, uint32_t *size)
823 uint32_t size_remaining = *size;
824 SVGA3dCmdHeader *header = (SVGA3dCmdHeader *) buf;
827 cmd_id = le32_to_cpu(((uint32_t *)buf)[0]);
828 /* Handle any none 3D commands */
829 if (unlikely(cmd_id < SVGA_CMD_MAX))
830 return vmw_cmd_check_not_3d(dev_priv, sw_context, buf, size);
833 cmd_id = le32_to_cpu(header->id);
834 *size = le32_to_cpu(header->size) + sizeof(SVGA3dCmdHeader);
836 cmd_id -= SVGA_3D_CMD_BASE;
837 if (unlikely(*size > size_remaining))
840 if (unlikely(cmd_id >= SVGA_3D_CMD_MAX - SVGA_3D_CMD_BASE))
843 ret = vmw_cmd_funcs[cmd_id](dev_priv, sw_context, header);
844 if (unlikely(ret != 0))
849 DRM_ERROR("Illegal / Invalid SVGA3D command: %d\n",
850 cmd_id + SVGA_3D_CMD_BASE);
854 static int vmw_cmd_check_all(struct vmw_private *dev_priv,
855 struct vmw_sw_context *sw_context,
859 int32_t cur_size = size;
862 while (cur_size > 0) {
864 ret = vmw_cmd_check(dev_priv, sw_context, buf, &size);
865 if (unlikely(ret != 0))
867 buf = (void *)((unsigned long) buf + size);
871 if (unlikely(cur_size != 0)) {
872 DRM_ERROR("Command verifier out of sync.\n");
879 static void vmw_free_relocations(struct vmw_sw_context *sw_context)
881 sw_context->cur_reloc = 0;
884 static void vmw_apply_relocations(struct vmw_sw_context *sw_context)
887 struct vmw_relocation *reloc;
888 struct ttm_validate_buffer *validate;
889 struct ttm_buffer_object *bo;
891 for (i = 0; i < sw_context->cur_reloc; ++i) {
892 reloc = &sw_context->relocs[i];
893 validate = &sw_context->val_bufs[reloc->index];
895 if (bo->mem.mem_type == TTM_PL_VRAM) {
896 reloc->location->offset += bo->offset;
897 reloc->location->gmrId = SVGA_GMR_FRAMEBUFFER;
899 reloc->location->gmrId = bo->mem.start;
901 vmw_free_relocations(sw_context);
904 static void vmw_clear_validations(struct vmw_sw_context *sw_context)
906 struct ttm_validate_buffer *entry, *next;
907 struct vmw_resource *res, *res_next;
910 * Drop references to DMA buffers held during command submission.
912 list_for_each_entry_safe(entry, next, &sw_context->validate_nodes,
914 list_del(&entry->head);
915 vmw_dmabuf_validate_clear(entry->bo);
916 ttm_bo_unref(&entry->bo);
917 sw_context->cur_val_buf--;
919 BUG_ON(sw_context->cur_val_buf != 0);
922 * Drop references to resources held during command submission.
924 vmw_resource_unreserve(&sw_context->resource_list);
925 list_for_each_entry_safe(res, res_next, &sw_context->resource_list,
927 list_del_init(&res->validate_head);
928 vmw_resource_unreference(&res);
932 static int vmw_validate_single_buffer(struct vmw_private *dev_priv,
933 struct ttm_buffer_object *bo)
939 * Don't validate pinned buffers.
942 if (bo == dev_priv->pinned_bo ||
943 (bo == dev_priv->dummy_query_bo &&
944 dev_priv->dummy_query_bo_pinned))
948 * Put BO in VRAM if there is space, otherwise as a GMR.
949 * If there is no space in VRAM and GMR ids are all used up,
950 * start evicting GMRs to make room. If the DMA buffer can't be
951 * used as a GMR, this will return -ENOMEM.
954 ret = ttm_bo_validate(bo, &vmw_vram_gmr_placement, true, false, false);
955 if (likely(ret == 0 || ret == -ERESTARTSYS))
959 * If that failed, try VRAM again, this time evicting
963 DRM_INFO("Falling through to VRAM.\n");
964 ret = ttm_bo_validate(bo, &vmw_vram_placement, true, false, false);
969 static int vmw_validate_buffers(struct vmw_private *dev_priv,
970 struct vmw_sw_context *sw_context)
972 struct ttm_validate_buffer *entry;
975 list_for_each_entry(entry, &sw_context->validate_nodes, head) {
976 ret = vmw_validate_single_buffer(dev_priv, entry->bo);
977 if (unlikely(ret != 0))
983 static int vmw_resize_cmd_bounce(struct vmw_sw_context *sw_context,
986 if (likely(sw_context->cmd_bounce_size >= size))
989 if (sw_context->cmd_bounce_size == 0)
990 sw_context->cmd_bounce_size = VMWGFX_CMD_BOUNCE_INIT_SIZE;
992 while (sw_context->cmd_bounce_size < size) {
993 sw_context->cmd_bounce_size =
994 PAGE_ALIGN(sw_context->cmd_bounce_size +
995 (sw_context->cmd_bounce_size >> 1));
998 if (sw_context->cmd_bounce != NULL)
999 vfree(sw_context->cmd_bounce);
1001 sw_context->cmd_bounce = vmalloc(sw_context->cmd_bounce_size);
1003 if (sw_context->cmd_bounce == NULL) {
1004 DRM_ERROR("Failed to allocate command bounce buffer.\n");
1005 sw_context->cmd_bounce_size = 0;
1013 * vmw_execbuf_fence_commands - create and submit a command stream fence
1015 * Creates a fence object and submits a command stream marker.
1016 * If this fails for some reason, We sync the fifo and return NULL.
1017 * It is then safe to fence buffers with a NULL pointer.
1019 * If @p_handle is not NULL @file_priv must also not be NULL. Creates
1020 * a userspace handle if @p_handle is not NULL, otherwise not.
1023 int vmw_execbuf_fence_commands(struct drm_file *file_priv,
1024 struct vmw_private *dev_priv,
1025 struct vmw_fence_obj **p_fence,
1030 bool synced = false;
1032 /* p_handle implies file_priv. */
1033 BUG_ON(p_handle != NULL && file_priv == NULL);
1035 ret = vmw_fifo_send_fence(dev_priv, &sequence);
1036 if (unlikely(ret != 0)) {
1037 DRM_ERROR("Fence submission error. Syncing.\n");
1041 if (p_handle != NULL)
1042 ret = vmw_user_fence_create(file_priv, dev_priv->fman,
1044 DRM_VMW_FENCE_FLAG_EXEC,
1047 ret = vmw_fence_create(dev_priv->fman, sequence,
1048 DRM_VMW_FENCE_FLAG_EXEC,
1051 if (unlikely(ret != 0 && !synced)) {
1052 (void) vmw_fallback_wait(dev_priv, false, false,
1054 VMW_FENCE_WAIT_TIMEOUT);
1062 * vmw_execbuf_copy_fence_user - copy fence object information to
1065 * @dev_priv: Pointer to a vmw_private struct.
1066 * @vmw_fp: Pointer to the struct vmw_fpriv representing the calling file.
1067 * @ret: Return value from fence object creation.
1068 * @user_fence_rep: User space address of a struct drm_vmw_fence_rep to
1069 * which the information should be copied.
1070 * @fence: Pointer to the fenc object.
1071 * @fence_handle: User-space fence handle.
1073 * This function copies fence information to user-space. If copying fails,
1074 * The user-space struct drm_vmw_fence_rep::error member is hopefully
1075 * left untouched, and if it's preloaded with an -EFAULT by user-space,
1076 * the error will hopefully be detected.
1077 * Also if copying fails, user-space will be unable to signal the fence
1078 * object so we wait for it immediately, and then unreference the
1079 * user-space reference.
1082 vmw_execbuf_copy_fence_user(struct vmw_private *dev_priv,
1083 struct vmw_fpriv *vmw_fp,
1085 struct drm_vmw_fence_rep __user *user_fence_rep,
1086 struct vmw_fence_obj *fence,
1087 uint32_t fence_handle)
1089 struct drm_vmw_fence_rep fence_rep;
1091 if (user_fence_rep == NULL)
1094 memset(&fence_rep, 0, sizeof(fence_rep));
1096 fence_rep.error = ret;
1098 BUG_ON(fence == NULL);
1100 fence_rep.handle = fence_handle;
1101 fence_rep.seqno = fence->seqno;
1102 vmw_update_seqno(dev_priv, &dev_priv->fifo);
1103 fence_rep.passed_seqno = dev_priv->last_read_seqno;
1107 * copy_to_user errors will be detected by user space not
1108 * seeing fence_rep::error filled in. Typically
1109 * user-space would have pre-set that member to -EFAULT.
1111 ret = copy_to_user(user_fence_rep, &fence_rep,
1115 * User-space lost the fence object. We need to sync
1116 * and unreference the handle.
1118 if (unlikely(ret != 0) && (fence_rep.error == 0)) {
1119 ttm_ref_object_base_unref(vmw_fp->tfile,
1120 fence_handle, TTM_REF_USAGE);
1121 DRM_ERROR("Fence copy error. Syncing.\n");
1122 (void) vmw_fence_obj_wait(fence, fence->signal_mask,
1124 VMW_FENCE_WAIT_TIMEOUT);
1128 int vmw_execbuf_process(struct drm_file *file_priv,
1129 struct vmw_private *dev_priv,
1130 void __user *user_commands,
1131 void *kernel_commands,
1132 uint32_t command_size,
1133 uint64_t throttle_us,
1134 struct drm_vmw_fence_rep __user *user_fence_rep)
1136 struct vmw_sw_context *sw_context = &dev_priv->ctx;
1137 struct vmw_fence_obj *fence;
1142 ret = mutex_lock_interruptible(&dev_priv->cmdbuf_mutex);
1143 if (unlikely(ret != 0))
1144 return -ERESTARTSYS;
1146 if (kernel_commands == NULL) {
1147 sw_context->kernel = false;
1149 ret = vmw_resize_cmd_bounce(sw_context, command_size);
1150 if (unlikely(ret != 0))
1154 ret = copy_from_user(sw_context->cmd_bounce,
1155 user_commands, command_size);
1157 if (unlikely(ret != 0)) {
1159 DRM_ERROR("Failed copying commands.\n");
1162 kernel_commands = sw_context->cmd_bounce;
1164 sw_context->kernel = true;
1166 sw_context->tfile = vmw_fpriv(file_priv)->tfile;
1167 sw_context->cid_valid = false;
1168 sw_context->sid_valid = false;
1169 sw_context->cur_reloc = 0;
1170 sw_context->cur_val_buf = 0;
1171 sw_context->fence_flags = 0;
1172 INIT_LIST_HEAD(&sw_context->query_list);
1173 INIT_LIST_HEAD(&sw_context->resource_list);
1174 sw_context->cur_query_bo = dev_priv->pinned_bo;
1175 sw_context->cur_query_cid = dev_priv->query_cid;
1176 sw_context->query_cid_valid = (dev_priv->pinned_bo != NULL);
1178 INIT_LIST_HEAD(&sw_context->validate_nodes);
1180 ret = vmw_cmd_check_all(dev_priv, sw_context, kernel_commands,
1182 if (unlikely(ret != 0))
1185 ret = ttm_eu_reserve_buffers(&sw_context->validate_nodes);
1186 if (unlikely(ret != 0))
1189 ret = vmw_validate_buffers(dev_priv, sw_context);
1190 if (unlikely(ret != 0))
1193 vmw_apply_relocations(sw_context);
1196 ret = vmw_wait_lag(dev_priv, &dev_priv->fifo.marker_queue,
1199 if (unlikely(ret != 0))
1203 cmd = vmw_fifo_reserve(dev_priv, command_size);
1204 if (unlikely(cmd == NULL)) {
1205 DRM_ERROR("Failed reserving fifo space for commands.\n");
1210 memcpy(cmd, kernel_commands, command_size);
1211 vmw_fifo_commit(dev_priv, command_size);
1213 vmw_query_bo_switch_commit(dev_priv, sw_context);
1214 ret = vmw_execbuf_fence_commands(file_priv, dev_priv,
1216 (user_fence_rep) ? &handle : NULL);
1218 * This error is harmless, because if fence submission fails,
1219 * vmw_fifo_send_fence will sync. The error will be propagated to
1220 * user-space in @fence_rep
1224 DRM_ERROR("Fence submission error. Syncing.\n");
1226 ttm_eu_fence_buffer_objects(&sw_context->validate_nodes,
1229 vmw_clear_validations(sw_context);
1230 vmw_execbuf_copy_fence_user(dev_priv, vmw_fpriv(file_priv), ret,
1231 user_fence_rep, fence, handle);
1233 if (likely(fence != NULL))
1234 vmw_fence_obj_unreference(&fence);
1236 mutex_unlock(&dev_priv->cmdbuf_mutex);
1240 vmw_free_relocations(sw_context);
1242 vmw_query_switch_backoff(sw_context);
1243 ttm_eu_backoff_reservation(&sw_context->validate_nodes);
1244 vmw_clear_validations(sw_context);
1246 mutex_unlock(&dev_priv->cmdbuf_mutex);
1251 * vmw_execbuf_unpin_panic - Idle the fifo and unpin the query buffer.
1253 * @dev_priv: The device private structure.
1255 * This function is called to idle the fifo and unpin the query buffer
1256 * if the normal way to do this hits an error, which should typically be
1259 static void vmw_execbuf_unpin_panic(struct vmw_private *dev_priv)
1261 DRM_ERROR("Can't unpin query buffer. Trying to recover.\n");
1263 (void) vmw_fallback_wait(dev_priv, false, true, 0, false, 10*HZ);
1264 vmw_bo_pin(dev_priv->pinned_bo, false);
1265 vmw_bo_pin(dev_priv->dummy_query_bo, false);
1266 dev_priv->dummy_query_bo_pinned = false;
1271 * vmw_execbuf_release_pinned_bo - Flush queries and unpin the pinned
1274 * @dev_priv: The device private structure.
1275 * @only_on_cid_match: Only flush and unpin if the current active query cid
1277 * @cid: Optional context id to match.
1279 * This function should be used to unpin the pinned query bo, or
1280 * as a query barrier when we need to make sure that all queries have
1281 * finished before the next fifo command. (For example on hardware
1282 * context destructions where the hardware may otherwise leak unfinished
1285 * This function does not return any failure codes, but make attempts
1286 * to do safe unpinning in case of errors.
1288 * The function will synchronize on the previous query barrier, and will
1289 * thus not finish until that barrier has executed.
1291 void vmw_execbuf_release_pinned_bo(struct vmw_private *dev_priv,
1292 bool only_on_cid_match, uint32_t cid)
1295 struct list_head validate_list;
1296 struct ttm_validate_buffer pinned_val, query_val;
1297 struct vmw_fence_obj *fence;
1299 mutex_lock(&dev_priv->cmdbuf_mutex);
1301 if (dev_priv->pinned_bo == NULL)
1304 if (only_on_cid_match && cid != dev_priv->query_cid)
1307 INIT_LIST_HEAD(&validate_list);
1309 pinned_val.new_sync_obj_arg = (void *)(unsigned long)
1310 DRM_VMW_FENCE_FLAG_EXEC;
1311 pinned_val.bo = ttm_bo_reference(dev_priv->pinned_bo);
1312 list_add_tail(&pinned_val.head, &validate_list);
1314 query_val.new_sync_obj_arg = pinned_val.new_sync_obj_arg;
1315 query_val.bo = ttm_bo_reference(dev_priv->dummy_query_bo);
1316 list_add_tail(&query_val.head, &validate_list);
1319 ret = ttm_eu_reserve_buffers(&validate_list);
1320 } while (ret == -ERESTARTSYS);
1322 if (unlikely(ret != 0)) {
1323 vmw_execbuf_unpin_panic(dev_priv);
1324 goto out_no_reserve;
1327 ret = vmw_fifo_emit_dummy_query(dev_priv, dev_priv->query_cid);
1328 if (unlikely(ret != 0)) {
1329 vmw_execbuf_unpin_panic(dev_priv);
1333 vmw_bo_pin(dev_priv->pinned_bo, false);
1334 vmw_bo_pin(dev_priv->dummy_query_bo, false);
1335 dev_priv->dummy_query_bo_pinned = false;
1337 (void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
1338 ttm_eu_fence_buffer_objects(&validate_list, (void *) fence);
1340 ttm_bo_unref(&query_val.bo);
1341 ttm_bo_unref(&pinned_val.bo);
1342 ttm_bo_unref(&dev_priv->pinned_bo);
1345 mutex_unlock(&dev_priv->cmdbuf_mutex);
1349 ttm_eu_backoff_reservation(&validate_list);
1351 ttm_bo_unref(&query_val.bo);
1352 ttm_bo_unref(&pinned_val.bo);
1353 ttm_bo_unref(&dev_priv->pinned_bo);
1354 mutex_unlock(&dev_priv->cmdbuf_mutex);
1358 int vmw_execbuf_ioctl(struct drm_device *dev, void *data,
1359 struct drm_file *file_priv)
1361 struct vmw_private *dev_priv = vmw_priv(dev);
1362 struct drm_vmw_execbuf_arg *arg = (struct drm_vmw_execbuf_arg *)data;
1363 struct vmw_master *vmaster = vmw_master(file_priv->master);
1367 * This will allow us to extend the ioctl argument while
1368 * maintaining backwards compatibility:
1369 * We take different code paths depending on the value of
1373 if (unlikely(arg->version != DRM_VMW_EXECBUF_VERSION)) {
1374 DRM_ERROR("Incorrect execbuf version.\n");
1375 DRM_ERROR("You're running outdated experimental "
1376 "vmwgfx user-space drivers.");
1380 ret = ttm_read_lock(&vmaster->lock, true);
1381 if (unlikely(ret != 0))
1384 ret = vmw_execbuf_process(file_priv, dev_priv,
1385 (void __user *)(unsigned long)arg->commands,
1386 NULL, arg->command_size, arg->throttle_us,
1387 (void __user *)(unsigned long)arg->fence_rep);
1389 if (unlikely(ret != 0))
1392 vmw_kms_cursor_post_execbuf(dev_priv);
1395 ttm_read_unlock(&vmaster->lock);