drivers/gpu/drm/exynos/exynos_drm_g2d.c

   1 /*
   2  * Copyright (C) 2012 Samsung Electronics Co.Ltd
   3  * Authors: Joonyoung Shim <jy0922.shim@samsung.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundationr
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/module.h>
  12 #include <linux/clk.h>
  13 #include <linux/err.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/io.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/pm_runtime.h>
  18 #include <linux/slab.h>
  19 #include <linux/workqueue.h>
  20 #include <linux/dma-mapping.h>
  21 #include <linux/dma-attrs.h>
  22
  23 #include <drm/drmP.h>
  24 #include <drm/exynos_drm.h>
  25 #include "exynos_drm_drv.h"
  26 #include "exynos_drm_gem.h"
  27 #include "exynos_drm_iommu.h"
  28
  29 #define G2D_HW_MAJOR_VER                4
  30 #define G2D_HW_MINOR_VER                1
  31
  32 /* vaild register range set from user: 0x0104 ~ 0x0880 */
  33 #define G2D_VALID_START                 0x0104
  34 #define G2D_VALID_END                   0x0880
  35
  36 /* general registers */
  37 #define G2D_SOFT_RESET                  0x0000
  38 #define G2D_INTEN                       0x0004
  39 #define G2D_INTC_PEND                   0x000C
  40 #define G2D_DMA_SFR_BASE_ADDR           0x0080
  41 #define G2D_DMA_COMMAND                 0x0084
  42 #define G2D_DMA_STATUS                  0x008C
  43 #define G2D_DMA_HOLD_CMD                0x0090
  44
  45 /* command registers */
  46 #define G2D_BITBLT_START                0x0100
  47
  48 /* registers for base address */
  49 #define G2D_SRC_BASE_ADDR               0x0304
  50 #define G2D_SRC_PLANE2_BASE_ADDR        0x0318
  51 #define G2D_DST_BASE_ADDR               0x0404
  52 #define G2D_DST_PLANE2_BASE_ADDR        0x0418
  53 #define G2D_PAT_BASE_ADDR               0x0500
  54 #define G2D_MSK_BASE_ADDR               0x0520
  55
  56 /* G2D_SOFT_RESET */
  57 #define G2D_SFRCLEAR                    (1 << 1)
  58 #define G2D_R                           (1 << 0)
  59
  60 /* G2D_INTEN */
  61 #define G2D_INTEN_ACF                   (1 << 3)
  62 #define G2D_INTEN_UCF                   (1 << 2)
  63 #define G2D_INTEN_GCF                   (1 << 1)
  64 #define G2D_INTEN_SCF                   (1 << 0)
  65
  66 /* G2D_INTC_PEND */
  67 #define G2D_INTP_ACMD_FIN               (1 << 3)
  68 #define G2D_INTP_UCMD_FIN               (1 << 2)
  69 #define G2D_INTP_GCMD_FIN               (1 << 1)
  70 #define G2D_INTP_SCMD_FIN               (1 << 0)
  71
  72 /* G2D_DMA_COMMAND */
  73 #define G2D_DMA_HALT                    (1 << 2)
  74 #define G2D_DMA_CONTINUE                (1 << 1)
  75 #define G2D_DMA_START                   (1 << 0)
  76
  77 /* G2D_DMA_STATUS */
  78 #define G2D_DMA_LIST_DONE_COUNT         (0xFF << 17)
  79 #define G2D_DMA_BITBLT_DONE_COUNT       (0xFFFF << 1)
  80 #define G2D_DMA_DONE                    (1 << 0)
  81 #define G2D_DMA_LIST_DONE_COUNT_OFFSET  17
  82
  83 /* G2D_DMA_HOLD_CMD */
  84 #define G2D_USET_HOLD                   (1 << 2)
  85 #define G2D_LIST_HOLD                   (1 << 1)
  86 #define G2D_BITBLT_HOLD                 (1 << 0)
  87
  88 /* G2D_BITBLT_START */
  89 #define G2D_START_CASESEL               (1 << 2)
  90 #define G2D_START_NHOLT                 (1 << 1)
  91 #define G2D_START_BITBLT                (1 << 0)
  92
  93 #define G2D_CMDLIST_SIZE                (PAGE_SIZE / 4)
  94 #define G2D_CMDLIST_NUM                 64
  95 #define G2D_CMDLIST_POOL_SIZE           (G2D_CMDLIST_SIZE * G2D_CMDLIST_NUM)
  96 #define G2D_CMDLIST_DATA_NUM            (G2D_CMDLIST_SIZE / sizeof(u32) - 2)
  97
  98 #define MAX_BUF_ADDR_NR                 6
  99
 100 /* maximum buffer pool size of userptr is 64MB as default */
 101 #define MAX_POOL                (64 * 1024 * 1024)
 102
 103 enum {
 104         BUF_TYPE_GEM = 1,
 105         BUF_TYPE_USERPTR,
 106 };
 107
 108 /* cmdlist data structure */
 109 struct g2d_cmdlist {
 110         u32             head;
 111         unsigned long   data[G2D_CMDLIST_DATA_NUM];
 112         u32             last;   /* last data offset */
 113 };
 114
 115 struct drm_exynos_pending_g2d_event {
 116         struct drm_pending_event        base;
 117         struct drm_exynos_g2d_event     event;
 118 };
 119
 120 struct g2d_cmdlist_userptr {
 121         struct list_head        list;
 122         dma_addr_t              dma_addr;
 123         unsigned long           userptr;
 124         unsigned long           size;
 125         struct page             **pages;
 126         unsigned int            npages;
 127         struct sg_table         *sgt;
 128         struct vm_area_struct   *vma;
 129         atomic_t                refcount;
 130         bool                    in_pool;
 131         bool                    out_of_list;
 132 };
 133
 134 struct g2d_cmdlist_node {
 135         struct list_head        list;
 136         struct g2d_cmdlist      *cmdlist;
 137         unsigned int            map_nr;
 138         unsigned long           handles[MAX_BUF_ADDR_NR];
 139         unsigned int            obj_type[MAX_BUF_ADDR_NR];
 140         dma_addr_t              dma_addr;
 141
 142         struct drm_exynos_pending_g2d_event     *event;
 143 };
 144
 145 struct g2d_runqueue_node {
 146         struct list_head        list;
 147         struct list_head        run_cmdlist;
 148         struct list_head        event_list;
 149         struct drm_file         *filp;
 150         pid_t                   pid;
 151         struct completion       complete;
 152         int                     async;
 153 };
 154
 155 struct g2d_data {
 156         struct device                   *dev;
 157         struct clk                      *gate_clk;
 158         void __iomem                    *regs;
 159         int                             irq;
 160         struct workqueue_struct         *g2d_workq;
 161         struct work_struct              runqueue_work;
 162         struct exynos_drm_subdrv        subdrv;
 163         bool                            suspended;
 164
 165         /* cmdlist */
 166         struct g2d_cmdlist_node         *cmdlist_node;
 167         struct list_head                free_cmdlist;
 168         struct mutex                    cmdlist_mutex;
 169         dma_addr_t                      cmdlist_pool;
 170         void                            *cmdlist_pool_virt;
 171         struct dma_attrs                cmdlist_dma_attrs;
 172
 173         /* runqueue*/
 174         struct g2d_runqueue_node        *runqueue_node;
 175         struct list_head                runqueue;
 176         struct mutex                    runqueue_mutex;
 177         struct kmem_cache               *runqueue_slab;
 178
 179         unsigned long                   current_pool;
 180         unsigned long                   max_pool;
 181 };
 182
 183 static int g2d_init_cmdlist(struct g2d_data *g2d)
 184 {
 185         struct device *dev = g2d->dev;
 186         struct g2d_cmdlist_node *node = g2d->cmdlist_node;
 187         struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
 188         int nr;
 189         int ret;
 190
 191         init_dma_attrs(&g2d->cmdlist_dma_attrs);
 192         dma_set_attr(DMA_ATTR_WRITE_COMBINE, &g2d->cmdlist_dma_attrs);
 193
 194         g2d->cmdlist_pool_virt = dma_alloc_attrs(subdrv->drm_dev->dev,
 195                                                 G2D_CMDLIST_POOL_SIZE,
 196                                                 &g2d->cmdlist_pool, GFP_KERNEL,
 197                                                 &g2d->cmdlist_dma_attrs);
 198         if (!g2d->cmdlist_pool_virt) {
 199                 dev_err(dev, "failed to allocate dma memory\n");
 200                 return -ENOMEM;
 201         }
 202
 203         node = kcalloc(G2D_CMDLIST_NUM, sizeof(*node), GFP_KERNEL);
 204         if (!node) {
 205                 dev_err(dev, "failed to allocate memory\n");
 206                 ret = -ENOMEM;
 207                 goto err;
 208         }
 209
 210         for (nr = 0; nr < G2D_CMDLIST_NUM; nr++) {
 211                 node[nr].cmdlist =
 212                         g2d->cmdlist_pool_virt + nr * G2D_CMDLIST_SIZE;
 213                 node[nr].dma_addr =
 214                         g2d->cmdlist_pool + nr * G2D_CMDLIST_SIZE;
 215
 216                 list_add_tail(&node[nr].list, &g2d->free_cmdlist);
 217         }
 218
 219         return 0;
 220
 221 err:
 222         dma_free_attrs(subdrv->drm_dev->dev, G2D_CMDLIST_POOL_SIZE,
 223                         g2d->cmdlist_pool_virt,
 224                         g2d->cmdlist_pool, &g2d->cmdlist_dma_attrs);
 225         return ret;
 226 }
 227
 228 static void g2d_fini_cmdlist(struct g2d_data *g2d)
 229 {
 230         struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
 231
 232         kfree(g2d->cmdlist_node);
 233         dma_free_attrs(subdrv->drm_dev->dev, G2D_CMDLIST_POOL_SIZE,
 234                         g2d->cmdlist_pool_virt,
 235                         g2d->cmdlist_pool, &g2d->cmdlist_dma_attrs);
 236 }
 237
 238 static struct g2d_cmdlist_node *g2d_get_cmdlist(struct g2d_data *g2d)
 239 {
 240         struct device *dev = g2d->dev;
 241         struct g2d_cmdlist_node *node;
 242
 243         mutex_lock(&g2d->cmdlist_mutex);
 244         if (list_empty(&g2d->free_cmdlist)) {
 245                 dev_err(dev, "there is no free cmdlist\n");
 246                 mutex_unlock(&g2d->cmdlist_mutex);
 247                 return NULL;
 248         }
 249
 250         node = list_first_entry(&g2d->free_cmdlist, struct g2d_cmdlist_node,
 251                                 list);
 252         list_del_init(&node->list);
 253         mutex_unlock(&g2d->cmdlist_mutex);
 254
 255         return node;
 256 }
 257
 258 static void g2d_put_cmdlist(struct g2d_data *g2d, struct g2d_cmdlist_node *node)
 259 {
 260         mutex_lock(&g2d->cmdlist_mutex);
 261         list_move_tail(&node->list, &g2d->free_cmdlist);
 262         mutex_unlock(&g2d->cmdlist_mutex);
 263 }
 264
 265 static void g2d_add_cmdlist_to_inuse(struct exynos_drm_g2d_private *g2d_priv,
 266                                      struct g2d_cmdlist_node *node)
 267 {
 268         struct g2d_cmdlist_node *lnode;
 269
 270         if (list_empty(&g2d_priv->inuse_cmdlist))
 271                 goto add_to_list;
 272
 273         /* this links to base address of new cmdlist */
 274         lnode = list_entry(g2d_priv->inuse_cmdlist.prev,
 275                                 struct g2d_cmdlist_node, list);
 276         lnode->cmdlist->data[lnode->cmdlist->last] = node->dma_addr;
 277
 278 add_to_list:
 279         list_add_tail(&node->list, &g2d_priv->inuse_cmdlist);
 280
 281         if (node->event)
 282                 list_add_tail(&node->event->base.link, &g2d_priv->event_list);
 283 }
 284
 285 static void g2d_userptr_put_dma_addr(struct drm_device *drm_dev,
 286                                         unsigned long obj,
 287                                         bool force)
 288 {
 289         struct g2d_cmdlist_userptr *g2d_userptr =
 290                                         (struct g2d_cmdlist_userptr *)obj;
 291
 292         if (!obj)
 293                 return;
 294
 295         if (force)
 296                 goto out;
 297
 298         atomic_dec(&g2d_userptr->refcount);
 299
 300         if (atomic_read(&g2d_userptr->refcount) > 0)
 301                 return;
 302
 303         if (g2d_userptr->in_pool)
 304                 return;
 305
 306 out:
 307         exynos_gem_unmap_sgt_from_dma(drm_dev, g2d_userptr->sgt,
 308                                         DMA_BIDIRECTIONAL);
 309
 310         exynos_gem_put_pages_to_userptr(g2d_userptr->pages,
 311                                         g2d_userptr->npages,
 312                                         g2d_userptr->vma);
 313
 314         if (!g2d_userptr->out_of_list)
 315                 list_del_init(&g2d_userptr->list);
 316
 317         sg_free_table(g2d_userptr->sgt);
 318         kfree(g2d_userptr->sgt);
 319         g2d_userptr->sgt = NULL;
 320
 321         kfree(g2d_userptr->pages);
 322         g2d_userptr->pages = NULL;
 323         kfree(g2d_userptr);
 324         g2d_userptr = NULL;
 325 }
 326
 327 dma_addr_t *g2d_userptr_get_dma_addr(struct drm_device *drm_dev,
 328                                         unsigned long userptr,
 329                                         unsigned long size,
 330                                         struct drm_file *filp,
 331                                         unsigned long *obj)
 332 {
 333         struct drm_exynos_file_private *file_priv = filp->driver_priv;
 334         struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
 335         struct g2d_cmdlist_userptr *g2d_userptr;
 336         struct g2d_data *g2d;
 337         struct page **pages;
 338         struct sg_table *sgt;
 339         struct vm_area_struct *vma;
 340         unsigned long start, end;
 341         unsigned int npages, offset;
 342         int ret;
 343
 344         if (!size) {
 345                 DRM_ERROR("invalid userptr size.\n");
 346                 return ERR_PTR(-EINVAL);
 347         }
 348
 349         g2d = dev_get_drvdata(g2d_priv->dev);
 350
 351         /* check if userptr already exists in userptr_list. */
 352         list_for_each_entry(g2d_userptr, &g2d_priv->userptr_list, list) {
 353                 if (g2d_userptr->userptr == userptr) {
 354                         /*
 355                          * also check size because there could be same address
 356                          * and different size.
 357                          */
 358                         if (g2d_userptr->size == size) {
 359                                 atomic_inc(&g2d_userptr->refcount);
 360                                 *obj = (unsigned long)g2d_userptr;
 361
 362                                 return &g2d_userptr->dma_addr;
 363                         }
 364
 365                         /*
 366                          * at this moment, maybe g2d dma is accessing this
 367                          * g2d_userptr memory region so just remove this
 368                          * g2d_userptr object from userptr_list not to be
 369                          * referred again and also except it the userptr
 370                          * pool to be released after the dma access completion.
 371                          */
 372                         g2d_userptr->out_of_list = true;
 373                         g2d_userptr->in_pool = false;
 374                         list_del_init(&g2d_userptr->list);
 375
 376                         break;
 377                 }
 378         }
 379
 380         g2d_userptr = kzalloc(sizeof(*g2d_userptr), GFP_KERNEL);
 381         if (!g2d_userptr) {
 382                 DRM_ERROR("failed to allocate g2d_userptr.\n");
 383                 return ERR_PTR(-ENOMEM);
 384         }
 385
 386         atomic_set(&g2d_userptr->refcount, 1);
 387
 388         start = userptr & PAGE_MASK;
 389         offset = userptr & ~PAGE_MASK;
 390         end = PAGE_ALIGN(userptr + size);
 391         npages = (end - start) >> PAGE_SHIFT;
 392         g2d_userptr->npages = npages;
 393
 394         pages = kzalloc(npages * sizeof(struct page *), GFP_KERNEL);
 395         if (!pages) {
 396                 DRM_ERROR("failed to allocate pages.\n");
 397                 kfree(g2d_userptr);
 398                 return ERR_PTR(-ENOMEM);
 399         }
 400
 401         vma = find_vma(current->mm, userptr);
 402         if (!vma) {
 403                 DRM_ERROR("failed to get vm region.\n");
 404                 ret = -EFAULT;
 405                 goto err_free_pages;
 406         }
 407
 408         if (vma->vm_end < userptr + size) {
 409                 DRM_ERROR("vma is too small.\n");
 410                 ret = -EFAULT;
 411                 goto err_free_pages;
 412         }
 413
 414         g2d_userptr->vma = exynos_gem_get_vma(vma);
 415         if (!g2d_userptr->vma) {
 416                 DRM_ERROR("failed to copy vma.\n");
 417                 ret = -ENOMEM;
 418                 goto err_free_pages;
 419         }
 420
 421         g2d_userptr->size = size;
 422
 423         ret = exynos_gem_get_pages_from_userptr(start & PAGE_MASK,
 424                                                 npages, pages, vma);
 425         if (ret < 0) {
 426                 DRM_ERROR("failed to get user pages from userptr.\n");
 427                 goto err_put_vma;
 428         }
 429
 430         g2d_userptr->pages = pages;
 431
 432         sgt = kzalloc(sizeof *sgt, GFP_KERNEL);
 433         if (!sgt) {
 434                 DRM_ERROR("failed to allocate sg table.\n");
 435                 ret = -ENOMEM;
 436                 goto err_free_userptr;
 437         }
 438
 439         ret = sg_alloc_table_from_pages(sgt, pages, npages, offset,
 440                                         size, GFP_KERNEL);
 441         if (ret < 0) {
 442                 DRM_ERROR("failed to get sgt from pages.\n");
 443                 goto err_free_sgt;
 444         }
 445
 446         g2d_userptr->sgt = sgt;
 447
 448         ret = exynos_gem_map_sgt_with_dma(drm_dev, g2d_userptr->sgt,
 449                                                 DMA_BIDIRECTIONAL);
 450         if (ret < 0) {
 451                 DRM_ERROR("failed to map sgt with dma region.\n");
 452                 goto err_free_sgt;
 453         }
 454
 455         g2d_userptr->dma_addr = sgt->sgl[0].dma_address;
 456         g2d_userptr->userptr = userptr;
 457
 458         list_add_tail(&g2d_userptr->list, &g2d_priv->userptr_list);
 459
 460         if (g2d->current_pool + (npages << PAGE_SHIFT) < g2d->max_pool) {
 461                 g2d->current_pool += npages << PAGE_SHIFT;
 462                 g2d_userptr->in_pool = true;
 463         }
 464
 465         *obj = (unsigned long)g2d_userptr;
 466
 467         return &g2d_userptr->dma_addr;
 468
 469 err_free_sgt:
 470         sg_free_table(sgt);
 471         kfree(sgt);
 472         sgt = NULL;
 473
 474 err_free_userptr:
 475         exynos_gem_put_pages_to_userptr(g2d_userptr->pages,
 476                                         g2d_userptr->npages,
 477                                         g2d_userptr->vma);
 478
 479 err_put_vma:
 480         exynos_gem_put_vma(g2d_userptr->vma);
 481
 482 err_free_pages:
 483         kfree(pages);
 484         kfree(g2d_userptr);
 485         pages = NULL;
 486         g2d_userptr = NULL;
 487
 488         return ERR_PTR(ret);
 489 }
 490
 491 static void g2d_userptr_free_all(struct drm_device *drm_dev,
 492                                         struct g2d_data *g2d,
 493                                         struct drm_file *filp)
 494 {
 495         struct drm_exynos_file_private *file_priv = filp->driver_priv;
 496         struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
 497         struct g2d_cmdlist_userptr *g2d_userptr, *n;
 498
 499         list_for_each_entry_safe(g2d_userptr, n, &g2d_priv->userptr_list, list)
 500                 if (g2d_userptr->in_pool)
 501                         g2d_userptr_put_dma_addr(drm_dev,
 502                                                 (unsigned long)g2d_userptr,
 503                                                 true);
 504
 505         g2d->current_pool = 0;
 506 }
 507
 508 static int g2d_map_cmdlist_gem(struct g2d_data *g2d,
 509                                 struct g2d_cmdlist_node *node,
 510                                 struct drm_device *drm_dev,
 511                                 struct drm_file *file)
 512 {
 513         struct g2d_cmdlist *cmdlist = node->cmdlist;
 514         int offset;
 515         int i;
 516
 517         for (i = 0; i < node->map_nr; i++) {
 518                 unsigned long handle;
 519                 dma_addr_t *addr;
 520
 521                 offset = cmdlist->last - (i * 2 + 1);
 522                 handle = cmdlist->data[offset];
 523
 524                 if (node->obj_type[i] == BUF_TYPE_GEM) {
 525                         addr = exynos_drm_gem_get_dma_addr(drm_dev, handle,
 526                                                                 file);
 527                         if (IS_ERR(addr)) {
 528                                 node->map_nr = i;
 529                                 return -EFAULT;
 530                         }
 531                 } else {
 532                         struct drm_exynos_g2d_userptr g2d_userptr;
 533
 534                         if (copy_from_user(&g2d_userptr, (void __user *)handle,
 535                                 sizeof(struct drm_exynos_g2d_userptr))) {
 536                                 node->map_nr = i;
 537                                 return -EFAULT;
 538                         }
 539
 540                         addr = g2d_userptr_get_dma_addr(drm_dev,
 541                                                         g2d_userptr.userptr,
 542                                                         g2d_userptr.size,
 543                                                         file,
 544                                                         &handle);
 545                         if (IS_ERR(addr)) {
 546                                 node->map_nr = i;
 547                                 return -EFAULT;
 548                         }
 549                 }
 550
 551                 cmdlist->data[offset] = *addr;
 552                 node->handles[i] = handle;
 553         }
 554
 555         return 0;
 556 }
 557
 558 static void g2d_unmap_cmdlist_gem(struct g2d_data *g2d,
 559                                   struct g2d_cmdlist_node *node,
 560                                   struct drm_file *filp)
 561 {
 562         struct exynos_drm_subdrv *subdrv = &g2d->subdrv;
 563         int i;
 564
 565         for (i = 0; i < node->map_nr; i++) {
 566                 unsigned long handle = node->handles[i];
 567
 568                 if (node->obj_type[i] == BUF_TYPE_GEM)
 569                         exynos_drm_gem_put_dma_addr(subdrv->drm_dev, handle,
 570                                                         filp);
 571                 else
 572                         g2d_userptr_put_dma_addr(subdrv->drm_dev, handle,
 573                                                         false);
 574
 575                 node->handles[i] = 0;
 576         }
 577
 578         node->map_nr = 0;
 579 }
 580
 581 static void g2d_dma_start(struct g2d_data *g2d,
 582                           struct g2d_runqueue_node *runqueue_node)
 583 {
 584         struct g2d_cmdlist_node *node =
 585                                 list_first_entry(&runqueue_node->run_cmdlist,
 586                                                 struct g2d_cmdlist_node, list);
 587
 588         pm_runtime_get_sync(g2d->dev);
 589         clk_enable(g2d->gate_clk);
 590
 591         /* interrupt enable */
 592         writel_relaxed(G2D_INTEN_ACF | G2D_INTEN_UCF | G2D_INTEN_GCF,
 593                         g2d->regs + G2D_INTEN);
 594
 595         writel_relaxed(node->dma_addr, g2d->regs + G2D_DMA_SFR_BASE_ADDR);
 596         writel_relaxed(G2D_DMA_START, g2d->regs + G2D_DMA_COMMAND);
 597 }
 598
 599 static struct g2d_runqueue_node *g2d_get_runqueue_node(struct g2d_data *g2d)
 600 {
 601         struct g2d_runqueue_node *runqueue_node;
 602
 603         if (list_empty(&g2d->runqueue))
 604                 return NULL;
 605
 606         runqueue_node = list_first_entry(&g2d->runqueue,
 607                                          struct g2d_runqueue_node, list);
 608         list_del_init(&runqueue_node->list);
 609         return runqueue_node;
 610 }
 611
 612 static void g2d_free_runqueue_node(struct g2d_data *g2d,
 613                                    struct g2d_runqueue_node *runqueue_node)
 614 {
 615         struct g2d_cmdlist_node *node;
 616
 617         if (!runqueue_node)
 618                 return;
 619
 620         mutex_lock(&g2d->cmdlist_mutex);
 621         /*
 622          * commands in run_cmdlist have been completed so unmap all gem
 623          * objects in each command node so that they are unreferenced.
 624          */
 625         list_for_each_entry(node, &runqueue_node->run_cmdlist, list)
 626                 g2d_unmap_cmdlist_gem(g2d, node, runqueue_node->filp);
 627         list_splice_tail_init(&runqueue_node->run_cmdlist, &g2d->free_cmdlist);
 628         mutex_unlock(&g2d->cmdlist_mutex);
 629
 630         kmem_cache_free(g2d->runqueue_slab, runqueue_node);
 631 }
 632
 633 static void g2d_exec_runqueue(struct g2d_data *g2d)
 634 {
 635         g2d->runqueue_node = g2d_get_runqueue_node(g2d);
 636         if (g2d->runqueue_node)
 637                 g2d_dma_start(g2d, g2d->runqueue_node);
 638 }
 639
 640 static void g2d_runqueue_worker(struct work_struct *work)
 641 {
 642         struct g2d_data *g2d = container_of(work, struct g2d_data,
 643                                             runqueue_work);
 644
 645
 646         mutex_lock(&g2d->runqueue_mutex);
 647         clk_disable(g2d->gate_clk);
 648         pm_runtime_put_sync(g2d->dev);
 649
 650         complete(&g2d->runqueue_node->complete);
 651         if (g2d->runqueue_node->async)
 652                 g2d_free_runqueue_node(g2d, g2d->runqueue_node);
 653
 654         if (g2d->suspended)
 655                 g2d->runqueue_node = NULL;
 656         else
 657                 g2d_exec_runqueue(g2d);
 658         mutex_unlock(&g2d->runqueue_mutex);
 659 }
 660
 661 static void g2d_finish_event(struct g2d_data *g2d, u32 cmdlist_no)
 662 {
 663         struct drm_device *drm_dev = g2d->subdrv.drm_dev;
 664         struct g2d_runqueue_node *runqueue_node = g2d->runqueue_node;
 665         struct drm_exynos_pending_g2d_event *e;
 666         struct timeval now;
 667         unsigned long flags;
 668
 669         if (list_empty(&runqueue_node->event_list))
 670                 return;
 671
 672         e = list_first_entry(&runqueue_node->event_list,
 673                              struct drm_exynos_pending_g2d_event, base.link);
 674
 675         do_gettimeofday(&now);
 676         e->event.tv_sec = now.tv_sec;
 677         e->event.tv_usec = now.tv_usec;
 678         e->event.cmdlist_no = cmdlist_no;
 679
 680         spin_lock_irqsave(&drm_dev->event_lock, flags);
 681         list_move_tail(&e->base.link, &e->base.file_priv->event_list);
 682         wake_up_interruptible(&e->base.file_priv->event_wait);
 683         spin_unlock_irqrestore(&drm_dev->event_lock, flags);
 684 }
 685
 686 static irqreturn_t g2d_irq_handler(int irq, void *dev_id)
 687 {
 688         struct g2d_data *g2d = dev_id;
 689         u32 pending;
 690
 691         pending = readl_relaxed(g2d->regs + G2D_INTC_PEND);
 692         if (pending)
 693                 writel_relaxed(pending, g2d->regs + G2D_INTC_PEND);
 694
 695         if (pending & G2D_INTP_GCMD_FIN) {
 696                 u32 cmdlist_no = readl_relaxed(g2d->regs + G2D_DMA_STATUS);
 697
 698                 cmdlist_no = (cmdlist_no & G2D_DMA_LIST_DONE_COUNT) >>
 699                                                 G2D_DMA_LIST_DONE_COUNT_OFFSET;
 700
 701                 g2d_finish_event(g2d, cmdlist_no);
 702
 703                 writel_relaxed(0, g2d->regs + G2D_DMA_HOLD_CMD);
 704                 if (!(pending & G2D_INTP_ACMD_FIN)) {
 705                         writel_relaxed(G2D_DMA_CONTINUE,
 706                                         g2d->regs + G2D_DMA_COMMAND);
 707                 }
 708         }
 709
 710         if (pending & G2D_INTP_ACMD_FIN)
 711                 queue_work(g2d->g2d_workq, &g2d->runqueue_work);
 712
 713         return IRQ_HANDLED;
 714 }
 715
 716 static int g2d_check_reg_offset(struct device *dev,
 717                                 struct g2d_cmdlist_node *node,
 718                                 int nr, bool for_addr)
 719 {
 720         struct g2d_cmdlist *cmdlist = node->cmdlist;
 721         int reg_offset;
 722         int index;
 723         int i;
 724
 725         for (i = 0; i < nr; i++) {
 726                 index = cmdlist->last - 2 * (i + 1);
 727
 728                 if (for_addr) {
 729                         /* check userptr buffer type. */
 730                         reg_offset = (cmdlist->data[index] &
 731                                         ~0x7fffffff) >> 31;
 732                         if (reg_offset) {
 733                                 node->obj_type[i] = BUF_TYPE_USERPTR;
 734                                 cmdlist->data[index] &= ~G2D_BUF_USERPTR;
 735                         }
 736                 }
 737
 738                 reg_offset = cmdlist->data[index] & ~0xfffff000;
 739
 740                 if (reg_offset < G2D_VALID_START || reg_offset > G2D_VALID_END)
 741                         goto err;
 742                 if (reg_offset % 4)
 743                         goto err;
 744
 745                 switch (reg_offset) {
 746                 case G2D_SRC_BASE_ADDR:
 747                 case G2D_SRC_PLANE2_BASE_ADDR:
 748                 case G2D_DST_BASE_ADDR:
 749                 case G2D_DST_PLANE2_BASE_ADDR:
 750                 case G2D_PAT_BASE_ADDR:
 751                 case G2D_MSK_BASE_ADDR:
 752                         if (!for_addr)
 753                                 goto err;
 754
 755                         if (node->obj_type[i] != BUF_TYPE_USERPTR)
 756                                 node->obj_type[i] = BUF_TYPE_GEM;
 757                         break;
 758                 default:
 759                         if (for_addr)
 760                                 goto err;
 761                         break;
 762                 }
 763         }
 764
 765         return 0;
 766
 767 err:
 768         dev_err(dev, "Bad register offset: 0x%lx\n", cmdlist->data[index]);
 769         return -EINVAL;
 770 }
 771
 772 /* ioctl functions */
 773 int exynos_g2d_get_ver_ioctl(struct drm_device *drm_dev, void *data,
 774                              struct drm_file *file)
 775 {
 776         struct drm_exynos_g2d_get_ver *ver = data;
 777
 778         ver->major = G2D_HW_MAJOR_VER;
 779         ver->minor = G2D_HW_MINOR_VER;
 780
 781         return 0;
 782 }
 783 EXPORT_SYMBOL_GPL(exynos_g2d_get_ver_ioctl);
 784
 785 int exynos_g2d_set_cmdlist_ioctl(struct drm_device *drm_dev, void *data,
 786                                  struct drm_file *file)
 787 {
 788         struct drm_exynos_file_private *file_priv = file->driver_priv;
 789         struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
 790         struct device *dev = g2d_priv->dev;
 791         struct g2d_data *g2d;
 792         struct drm_exynos_g2d_set_cmdlist *req = data;
 793         struct drm_exynos_g2d_cmd *cmd;
 794         struct drm_exynos_pending_g2d_event *e;
 795         struct g2d_cmdlist_node *node;
 796         struct g2d_cmdlist *cmdlist;
 797         unsigned long flags;
 798         int size;
 799         int ret;
 800
 801         if (!dev)
 802                 return -ENODEV;
 803
 804         g2d = dev_get_drvdata(dev);
 805         if (!g2d)
 806                 return -EFAULT;
 807
 808         node = g2d_get_cmdlist(g2d);
 809         if (!node)
 810                 return -ENOMEM;
 811
 812         node->event = NULL;
 813
 814         if (req->event_type != G2D_EVENT_NOT) {
 815                 spin_lock_irqsave(&drm_dev->event_lock, flags);
 816                 if (file->event_space < sizeof(e->event)) {
 817                         spin_unlock_irqrestore(&drm_dev->event_lock, flags);
 818                         ret = -ENOMEM;
 819                         goto err;
 820                 }
 821                 file->event_space -= sizeof(e->event);
 822                 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
 823
 824                 e = kzalloc(sizeof(*node->event), GFP_KERNEL);
 825                 if (!e) {
 826                         dev_err(dev, "failed to allocate event\n");
 827
 828                         spin_lock_irqsave(&drm_dev->event_lock, flags);
 829                         file->event_space += sizeof(e->event);
 830                         spin_unlock_irqrestore(&drm_dev->event_lock, flags);
 831
 832                         ret = -ENOMEM;
 833                         goto err;
 834                 }
 835
 836                 e->event.base.type = DRM_EXYNOS_G2D_EVENT;
 837                 e->event.base.length = sizeof(e->event);
 838                 e->event.user_data = req->user_data;
 839                 e->base.event = &e->event.base;
 840                 e->base.file_priv = file;
 841                 e->base.destroy = (void (*) (struct drm_pending_event *)) kfree;
 842
 843                 node->event = e;
 844         }
 845
 846         cmdlist = node->cmdlist;
 847
 848         cmdlist->last = 0;
 849
 850         /*
 851          * If don't clear SFR registers, the cmdlist is affected by register
 852          * values of previous cmdlist. G2D hw executes SFR clear command and
 853          * a next command at the same time then the next command is ignored and
 854          * is executed rightly from next next command, so needs a dummy command
 855          * to next command of SFR clear command.
 856          */
 857         cmdlist->data[cmdlist->last++] = G2D_SOFT_RESET;
 858         cmdlist->data[cmdlist->last++] = G2D_SFRCLEAR;
 859         cmdlist->data[cmdlist->last++] = G2D_SRC_BASE_ADDR;
 860         cmdlist->data[cmdlist->last++] = 0;
 861
 862         if (node->event) {
 863                 cmdlist->data[cmdlist->last++] = G2D_DMA_HOLD_CMD;
 864                 cmdlist->data[cmdlist->last++] = G2D_LIST_HOLD;
 865         }
 866
 867         /* Check size of cmdlist: last 2 is about G2D_BITBLT_START */
 868         size = cmdlist->last + req->cmd_nr * 2 + req->cmd_buf_nr * 2 + 2;
 869         if (size > G2D_CMDLIST_DATA_NUM) {
 870                 dev_err(dev, "cmdlist size is too big\n");
 871                 ret = -EINVAL;
 872                 goto err_free_event;
 873         }
 874
 875         cmd = (struct drm_exynos_g2d_cmd *)(uint32_t)req->cmd;
 876
 877         if (copy_from_user(cmdlist->data + cmdlist->last,
 878                                 (void __user *)cmd,
 879                                 sizeof(*cmd) * req->cmd_nr)) {
 880                 ret = -EFAULT;
 881                 goto err_free_event;
 882         }
 883         cmdlist->last += req->cmd_nr * 2;
 884
 885         ret = g2d_check_reg_offset(dev, node, req->cmd_nr, false);
 886         if (ret < 0)
 887                 goto err_free_event;
 888
 889         node->map_nr = req->cmd_buf_nr;
 890         if (req->cmd_buf_nr) {
 891                 struct drm_exynos_g2d_cmd *cmd_buf;
 892
 893                 cmd_buf = (struct drm_exynos_g2d_cmd *)(uint32_t)req->cmd_buf;
 894
 895                 if (copy_from_user(cmdlist->data + cmdlist->last,
 896                                         (void __user *)cmd_buf,
 897                                         sizeof(*cmd_buf) * req->cmd_buf_nr)) {
 898                         ret = -EFAULT;
 899                         goto err_free_event;
 900                 }
 901                 cmdlist->last += req->cmd_buf_nr * 2;
 902
 903                 ret = g2d_check_reg_offset(dev, node, req->cmd_buf_nr, true);
 904                 if (ret < 0)
 905                         goto err_free_event;
 906
 907                 ret = g2d_map_cmdlist_gem(g2d, node, drm_dev, file);
 908                 if (ret < 0)
 909                         goto err_unmap;
 910         }
 911
 912         cmdlist->data[cmdlist->last++] = G2D_BITBLT_START;
 913         cmdlist->data[cmdlist->last++] = G2D_START_BITBLT;
 914
 915         /* head */
 916         cmdlist->head = cmdlist->last / 2;
 917
 918         /* tail */
 919         cmdlist->data[cmdlist->last] = 0;
 920
 921         g2d_add_cmdlist_to_inuse(g2d_priv, node);
 922
 923         return 0;
 924
 925 err_unmap:
 926         g2d_unmap_cmdlist_gem(g2d, node, file);
 927 err_free_event:
 928         if (node->event) {
 929                 spin_lock_irqsave(&drm_dev->event_lock, flags);
 930                 file->event_space += sizeof(e->event);
 931                 spin_unlock_irqrestore(&drm_dev->event_lock, flags);
 932                 kfree(node->event);
 933         }
 934 err:
 935         g2d_put_cmdlist(g2d, node);
 936         return ret;
 937 }
 938 EXPORT_SYMBOL_GPL(exynos_g2d_set_cmdlist_ioctl);
 939
 940 int exynos_g2d_exec_ioctl(struct drm_device *drm_dev, void *data,
 941                           struct drm_file *file)
 942 {
 943         struct drm_exynos_file_private *file_priv = file->driver_priv;
 944         struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
 945         struct device *dev = g2d_priv->dev;
 946         struct g2d_data *g2d;
 947         struct drm_exynos_g2d_exec *req = data;
 948         struct g2d_runqueue_node *runqueue_node;
 949         struct list_head *run_cmdlist;
 950         struct list_head *event_list;
 951
 952         if (!dev)
 953                 return -ENODEV;
 954
 955         g2d = dev_get_drvdata(dev);
 956         if (!g2d)
 957                 return -EFAULT;
 958
 959         runqueue_node = kmem_cache_alloc(g2d->runqueue_slab, GFP_KERNEL);
 960         if (!runqueue_node) {
 961                 dev_err(dev, "failed to allocate memory\n");
 962                 return -ENOMEM;
 963         }
 964         run_cmdlist = &runqueue_node->run_cmdlist;
 965         event_list = &runqueue_node->event_list;
 966         INIT_LIST_HEAD(run_cmdlist);
 967         INIT_LIST_HEAD(event_list);
 968         init_completion(&runqueue_node->complete);
 969         runqueue_node->async = req->async;
 970
 971         list_splice_init(&g2d_priv->inuse_cmdlist, run_cmdlist);
 972         list_splice_init(&g2d_priv->event_list, event_list);
 973
 974         if (list_empty(run_cmdlist)) {
 975                 dev_err(dev, "there is no inuse cmdlist\n");
 976                 kmem_cache_free(g2d->runqueue_slab, runqueue_node);
 977                 return -EPERM;
 978         }
 979
 980         mutex_lock(&g2d->runqueue_mutex);
 981         runqueue_node->pid = current->pid;
 982         runqueue_node->filp = file;
 983         list_add_tail(&runqueue_node->list, &g2d->runqueue);
 984         if (!g2d->runqueue_node)
 985                 g2d_exec_runqueue(g2d);
 986         mutex_unlock(&g2d->runqueue_mutex);
 987
 988         if (runqueue_node->async)
 989                 goto out;
 990
 991         wait_for_completion(&runqueue_node->complete);
 992         g2d_free_runqueue_node(g2d, runqueue_node);
 993
 994 out:
 995         return 0;
 996 }
 997 EXPORT_SYMBOL_GPL(exynos_g2d_exec_ioctl);
 998
 999 static int g2d_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
1000 {
1001         struct g2d_data *g2d;
1002         int ret;
1003
1004         g2d = dev_get_drvdata(dev);
1005         if (!g2d)
1006                 return -EFAULT;
1007
1008         /* allocate dma-aware cmdlist buffer. */
1009         ret = g2d_init_cmdlist(g2d);
1010         if (ret < 0) {
1011                 dev_err(dev, "cmdlist init failed\n");
1012                 return ret;
1013         }
1014
1015         if (!is_drm_iommu_supported(drm_dev))
1016                 return 0;
1017
1018         ret = drm_iommu_attach_device(drm_dev, dev);
1019         if (ret < 0) {
1020                 dev_err(dev, "failed to enable iommu.\n");
1021                 g2d_fini_cmdlist(g2d);
1022         }
1023
1024         return ret;
1025
1026 }
1027
1028 static void g2d_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
1029 {
1030         if (!is_drm_iommu_supported(drm_dev))
1031                 return;
1032
1033         drm_iommu_detach_device(drm_dev, dev);
1034 }
1035
1036 static int g2d_open(struct drm_device *drm_dev, struct device *dev,
1037                         struct drm_file *file)
1038 {
1039         struct drm_exynos_file_private *file_priv = file->driver_priv;
1040         struct exynos_drm_g2d_private *g2d_priv;
1041
1042         g2d_priv = kzalloc(sizeof(*g2d_priv), GFP_KERNEL);
1043         if (!g2d_priv) {
1044                 dev_err(dev, "failed to allocate g2d private data\n");
1045                 return -ENOMEM;
1046         }
1047
1048         g2d_priv->dev = dev;
1049         file_priv->g2d_priv = g2d_priv;
1050
1051         INIT_LIST_HEAD(&g2d_priv->inuse_cmdlist);
1052         INIT_LIST_HEAD(&g2d_priv->event_list);
1053         INIT_LIST_HEAD(&g2d_priv->userptr_list);
1054
1055         return 0;
1056 }
1057
1058 static void g2d_close(struct drm_device *drm_dev, struct device *dev,
1059                         struct drm_file *file)
1060 {
1061         struct drm_exynos_file_private *file_priv = file->driver_priv;
1062         struct exynos_drm_g2d_private *g2d_priv = file_priv->g2d_priv;
1063         struct g2d_data *g2d;
1064         struct g2d_cmdlist_node *node, *n;
1065
1066         if (!dev)
1067                 return;
1068
1069         g2d = dev_get_drvdata(dev);
1070         if (!g2d)
1071                 return;
1072
1073         mutex_lock(&g2d->cmdlist_mutex);
1074         list_for_each_entry_safe(node, n, &g2d_priv->inuse_cmdlist, list) {
1075                 /*
1076                  * unmap all gem objects not completed.
1077                  *
1078                  * P.S. if current process was terminated forcely then
1079                  * there may be some commands in inuse_cmdlist so unmap
1080                  * them.
1081                  */
1082                 g2d_unmap_cmdlist_gem(g2d, node, file);
1083                 list_move_tail(&node->list, &g2d->free_cmdlist);
1084         }
1085         mutex_unlock(&g2d->cmdlist_mutex);
1086
1087         /* release all g2d_userptr in pool. */
1088         g2d_userptr_free_all(drm_dev, g2d, file);
1089
1090         kfree(file_priv->g2d_priv);
1091 }
1092
1093 static int g2d_probe(struct platform_device *pdev)
1094 {
1095         struct device *dev = &pdev->dev;
1096         struct resource *res;
1097         struct g2d_data *g2d;
1098         struct exynos_drm_subdrv *subdrv;
1099         int ret;
1100
1101         g2d = devm_kzalloc(&pdev->dev, sizeof(*g2d), GFP_KERNEL);
1102         if (!g2d) {
1103                 dev_err(dev, "failed to allocate driver data\n");
1104                 return -ENOMEM;
1105         }
1106
1107         g2d->runqueue_slab = kmem_cache_create("g2d_runqueue_slab",
1108                         sizeof(struct g2d_runqueue_node), 0, 0, NULL);
1109         if (!g2d->runqueue_slab)
1110                 return -ENOMEM;
1111
1112         g2d->dev = dev;
1113
1114         g2d->g2d_workq = create_singlethread_workqueue("g2d");
1115         if (!g2d->g2d_workq) {
1116                 dev_err(dev, "failed to create workqueue\n");
1117                 ret = -EINVAL;
1118                 goto err_destroy_slab;
1119         }
1120
1121         INIT_WORK(&g2d->runqueue_work, g2d_runqueue_worker);
1122         INIT_LIST_HEAD(&g2d->free_cmdlist);
1123         INIT_LIST_HEAD(&g2d->runqueue);
1124
1125         mutex_init(&g2d->cmdlist_mutex);
1126         mutex_init(&g2d->runqueue_mutex);
1127
1128         g2d->gate_clk = devm_clk_get(dev, "fimg2d");
1129         if (IS_ERR(g2d->gate_clk)) {
1130                 dev_err(dev, "failed to get gate clock\n");
1131                 ret = PTR_ERR(g2d->gate_clk);
1132                 goto err_destroy_workqueue;
1133         }
1134
1135         pm_runtime_enable(dev);
1136
1137         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1138
1139         g2d->regs = devm_request_and_ioremap(&pdev->dev, res);
1140         if (!g2d->regs) {
1141                 dev_err(dev, "failed to remap I/O memory\n");
1142                 ret = -ENXIO;
1143                 goto err_put_clk;
1144         }
1145
1146         g2d->irq = platform_get_irq(pdev, 0);
1147         if (g2d->irq < 0) {
1148                 dev_err(dev, "failed to get irq\n");
1149                 ret = g2d->irq;
1150                 goto err_put_clk;
1151         }
1152
1153         ret = devm_request_irq(&pdev->dev, g2d->irq, g2d_irq_handler, 0,
1154                                                                 "drm_g2d", g2d);
1155         if (ret < 0) {
1156                 dev_err(dev, "irq request failed\n");
1157                 goto err_put_clk;
1158         }
1159
1160         g2d->max_pool = MAX_POOL;
1161
1162         platform_set_drvdata(pdev, g2d);
1163
1164         subdrv = &g2d->subdrv;
1165         subdrv->dev = dev;
1166         subdrv->probe = g2d_subdrv_probe;
1167         subdrv->remove = g2d_subdrv_remove;
1168         subdrv->open = g2d_open;
1169         subdrv->close = g2d_close;
1170
1171         ret = exynos_drm_subdrv_register(subdrv);
1172         if (ret < 0) {
1173                 dev_err(dev, "failed to register drm g2d device\n");
1174                 goto err_put_clk;
1175         }
1176
1177         dev_info(dev, "The exynos g2d(ver %d.%d) successfully probed\n",
1178                         G2D_HW_MAJOR_VER, G2D_HW_MINOR_VER);
1179
1180         return 0;
1181
1182 err_put_clk:
1183         pm_runtime_disable(dev);
1184 err_destroy_workqueue:
1185         destroy_workqueue(g2d->g2d_workq);
1186 err_destroy_slab:
1187         kmem_cache_destroy(g2d->runqueue_slab);
1188         return ret;
1189 }
1190
1191 static int g2d_remove(struct platform_device *pdev)
1192 {
1193         struct g2d_data *g2d = platform_get_drvdata(pdev);
1194
1195         cancel_work_sync(&g2d->runqueue_work);
1196         exynos_drm_subdrv_unregister(&g2d->subdrv);
1197
1198         while (g2d->runqueue_node) {
1199                 g2d_free_runqueue_node(g2d, g2d->runqueue_node);
1200                 g2d->runqueue_node = g2d_get_runqueue_node(g2d);
1201         }
1202
1203         pm_runtime_disable(&pdev->dev);
1204
1205         g2d_fini_cmdlist(g2d);
1206         destroy_workqueue(g2d->g2d_workq);
1207         kmem_cache_destroy(g2d->runqueue_slab);
1208
1209         return 0;
1210 }
1211
1212 #ifdef CONFIG_PM_SLEEP
1213 static int g2d_suspend(struct device *dev)
1214 {
1215         struct g2d_data *g2d = dev_get_drvdata(dev);
1216
1217         mutex_lock(&g2d->runqueue_mutex);
1218         g2d->suspended = true;
1219         mutex_unlock(&g2d->runqueue_mutex);
1220
1221         while (g2d->runqueue_node)
1222                 /* FIXME: good range? */
1223                 usleep_range(500, 1000);
1224
1225         flush_work(&g2d->runqueue_work);
1226
1227         return 0;
1228 }
1229
1230 static int g2d_resume(struct device *dev)
1231 {
1232         struct g2d_data *g2d = dev_get_drvdata(dev);
1233
1234         g2d->suspended = false;
1235         g2d_exec_runqueue(g2d);
1236
1237         return 0;
1238 }
1239 #endif
1240
1241 static SIMPLE_DEV_PM_OPS(g2d_pm_ops, g2d_suspend, g2d_resume);
1242
1243 struct platform_driver g2d_driver = {
1244         .probe          = g2d_probe,
1245         .remove         = g2d_remove,
1246         .driver         = {
1247                 .name   = "s5p-g2d",
1248                 .owner  = THIS_MODULE,
1249                 .pm     = &g2d_pm_ops,
1250         },
1251 };