drivers/gpu/drm/i915/i915_gem_gtt.c

   1 /*
   2  * Copyright © 2010 Daniel Vetter
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include "drmP.h"
  26 #include "drm.h"
  27 #include "i915_drm.h"
  28 #include "i915_drv.h"
  29 #include "i915_trace.h"
  30 #include "intel_drv.h"
  31
  32 /* PPGTT support for Sandybdrige/Gen6 and later */
  33 static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt,
  34                                    unsigned first_entry,
  35                                    unsigned num_entries)
  36 {
  37         uint32_t *pt_vaddr;
  38         uint32_t scratch_pte;
  39         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
  40         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
  41         unsigned last_pte, i;
  42
  43         scratch_pte = GEN6_PTE_ADDR_ENCODE(ppgtt->scratch_page_dma_addr);
  44         scratch_pte |= GEN6_PTE_VALID | GEN6_PTE_CACHE_LLC;
  45
  46         while (num_entries) {
  47                 last_pte = first_pte + num_entries;
  48                 if (last_pte > I915_PPGTT_PT_ENTRIES)
  49                         last_pte = I915_PPGTT_PT_ENTRIES;
  50
  51                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
  52
  53                 for (i = first_pte; i < last_pte; i++)
  54                         pt_vaddr[i] = scratch_pte;
  55
  56                 kunmap_atomic(pt_vaddr);
  57
  58                 num_entries -= last_pte - first_pte;
  59                 first_pte = 0;
  60                 act_pd++;
  61         }
  62 }
  63
  64 int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
  65 {
  66         struct drm_i915_private *dev_priv = dev->dev_private;
  67         struct i915_hw_ppgtt *ppgtt;
  68         uint32_t pd_entry;
  69         unsigned first_pd_entry_in_global_pt;
  70         uint32_t __iomem *pd_addr;
  71         int i;
  72         int ret = -ENOMEM;
  73
  74         /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
  75          * entries. For aliasing ppgtt support we just steal them at the end for
  76          * now. */
  77         first_pd_entry_in_global_pt = 512*1024 - I915_PPGTT_PD_ENTRIES;
  78
  79         ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
  80         if (!ppgtt)
  81                 return ret;
  82
  83         ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES;
  84         ppgtt->pt_pages = kzalloc(sizeof(struct page *)*ppgtt->num_pd_entries,
  85                                   GFP_KERNEL);
  86         if (!ppgtt->pt_pages)
  87                 goto err_ppgtt;
  88
  89         for (i = 0; i < ppgtt->num_pd_entries; i++) {
  90                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
  91                 if (!ppgtt->pt_pages[i])
  92                         goto err_pt_alloc;
  93         }
  94
  95         if (dev_priv->mm.gtt->needs_dmar) {
  96                 ppgtt->pt_dma_addr = kzalloc(sizeof(dma_addr_t)
  97                                                 *ppgtt->num_pd_entries,
  98                                              GFP_KERNEL);
  99                 if (!ppgtt->pt_dma_addr)
 100                         goto err_pt_alloc;
 101         }
 102
 103         pd_addr = dev_priv->mm.gtt->gtt + first_pd_entry_in_global_pt;
 104         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 105                 dma_addr_t pt_addr;
 106                 if (dev_priv->mm.gtt->needs_dmar) {
 107                         pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i],
 108                                                0, 4096,
 109                                                PCI_DMA_BIDIRECTIONAL);
 110
 111                         if (pci_dma_mapping_error(dev->pdev,
 112                                                   pt_addr)) {
 113                                 ret = -EIO;
 114                                 goto err_pd_pin;
 115
 116                         }
 117                         ppgtt->pt_dma_addr[i] = pt_addr;
 118                 } else
 119                         pt_addr = page_to_phys(ppgtt->pt_pages[i]);
 120
 121                 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
 122                 pd_entry |= GEN6_PDE_VALID;
 123
 124                 writel(pd_entry, pd_addr + i);
 125         }
 126         readl(pd_addr);
 127
 128         ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma;
 129
 130         i915_ppgtt_clear_range(ppgtt, 0,
 131                                ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES);
 132
 133         ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(uint32_t);
 134
 135         dev_priv->mm.aliasing_ppgtt = ppgtt;
 136
 137         return 0;
 138
 139 err_pd_pin:
 140         if (ppgtt->pt_dma_addr) {
 141                 for (i--; i >= 0; i--)
 142                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 143                                        4096, PCI_DMA_BIDIRECTIONAL);
 144         }
 145 err_pt_alloc:
 146         kfree(ppgtt->pt_dma_addr);
 147         for (i = 0; i < ppgtt->num_pd_entries; i++) {
 148                 if (ppgtt->pt_pages[i])
 149                         __free_page(ppgtt->pt_pages[i]);
 150         }
 151         kfree(ppgtt->pt_pages);
 152 err_ppgtt:
 153         kfree(ppgtt);
 154
 155         return ret;
 156 }
 157
 158 void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
 159 {
 160         struct drm_i915_private *dev_priv = dev->dev_private;
 161         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 162         int i;
 163
 164         if (!ppgtt)
 165                 return;
 166
 167         if (ppgtt->pt_dma_addr) {
 168                 for (i = 0; i < ppgtt->num_pd_entries; i++)
 169                         pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
 170                                        4096, PCI_DMA_BIDIRECTIONAL);
 171         }
 172
 173         kfree(ppgtt->pt_dma_addr);
 174         for (i = 0; i < ppgtt->num_pd_entries; i++)
 175                 __free_page(ppgtt->pt_pages[i]);
 176         kfree(ppgtt->pt_pages);
 177         kfree(ppgtt);
 178 }
 179
 180 static void i915_ppgtt_insert_sg_entries(struct i915_hw_ppgtt *ppgtt,
 181                                          struct scatterlist *sg_list,
 182                                          unsigned sg_len,
 183                                          unsigned first_entry,
 184                                          uint32_t pte_flags)
 185 {
 186         uint32_t *pt_vaddr, pte;
 187         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 188         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 189         unsigned i, j, m, segment_len;
 190         dma_addr_t page_addr;
 191         struct scatterlist *sg;
 192
 193         /* init sg walking */
 194         sg = sg_list;
 195         i = 0;
 196         segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 197         m = 0;
 198
 199         while (i < sg_len) {
 200                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 201
 202                 for (j = first_pte; j < I915_PPGTT_PT_ENTRIES; j++) {
 203                         page_addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 204                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 205                         pt_vaddr[j] = pte | pte_flags;
 206
 207                         /* grab the next page */
 208                         m++;
 209                         if (m == segment_len) {
 210                                 sg = sg_next(sg);
 211                                 i++;
 212                                 if (i == sg_len)
 213                                         break;
 214
 215                                 segment_len = sg_dma_len(sg) >> PAGE_SHIFT;
 216                                 m = 0;
 217                         }
 218                 }
 219
 220                 kunmap_atomic(pt_vaddr);
 221
 222                 first_pte = 0;
 223                 act_pd++;
 224         }
 225 }
 226
 227 static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt,
 228                                     unsigned first_entry, unsigned num_entries,
 229                                     struct page **pages, uint32_t pte_flags)
 230 {
 231         uint32_t *pt_vaddr, pte;
 232         unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES;
 233         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
 234         unsigned last_pte, i;
 235         dma_addr_t page_addr;
 236
 237         while (num_entries) {
 238                 last_pte = first_pte + num_entries;
 239                 last_pte = min_t(unsigned, last_pte, I915_PPGTT_PT_ENTRIES);
 240
 241                 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pd]);
 242
 243                 for (i = first_pte; i < last_pte; i++) {
 244                         page_addr = page_to_phys(*pages);
 245                         pte = GEN6_PTE_ADDR_ENCODE(page_addr);
 246                         pt_vaddr[i] = pte | pte_flags;
 247
 248                         pages++;
 249                 }
 250
 251                 kunmap_atomic(pt_vaddr);
 252
 253                 num_entries -= last_pte - first_pte;
 254                 first_pte = 0;
 255                 act_pd++;
 256         }
 257 }
 258
 259 void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
 260                             struct drm_i915_gem_object *obj,
 261                             enum i915_cache_level cache_level)
 262 {
 263         struct drm_device *dev = obj->base.dev;
 264         struct drm_i915_private *dev_priv = dev->dev_private;
 265         uint32_t pte_flags = GEN6_PTE_VALID;
 266
 267         switch (cache_level) {
 268         case I915_CACHE_LLC_MLC:
 269                 pte_flags |= GEN6_PTE_CACHE_LLC_MLC;
 270                 break;
 271         case I915_CACHE_LLC:
 272                 pte_flags |= GEN6_PTE_CACHE_LLC;
 273                 break;
 274         case I915_CACHE_NONE:
 275                 pte_flags |= GEN6_PTE_UNCACHED;
 276                 break;
 277         default:
 278                 BUG();
 279         }
 280
 281         if (dev_priv->mm.gtt->needs_dmar) {
 282                 BUG_ON(!obj->sg_list);
 283
 284                 i915_ppgtt_insert_sg_entries(ppgtt,
 285                                              obj->sg_list,
 286                                              obj->num_sg,
 287                                              obj->gtt_space->start >> PAGE_SHIFT,
 288                                              pte_flags);
 289         } else
 290                 i915_ppgtt_insert_pages(ppgtt,
 291                                         obj->gtt_space->start >> PAGE_SHIFT,
 292                                         obj->base.size >> PAGE_SHIFT,
 293                                         obj->pages,
 294                                         pte_flags);
 295 }
 296
 297 void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
 298                               struct drm_i915_gem_object *obj)
 299 {
 300         i915_ppgtt_clear_range(ppgtt,
 301                                obj->gtt_space->start >> PAGE_SHIFT,
 302                                obj->base.size >> PAGE_SHIFT);
 303 }
 304
 305 /* XXX kill agp_type! */
 306 static unsigned int cache_level_to_agp_type(struct drm_device *dev,
 307                                             enum i915_cache_level cache_level)
 308 {
 309         switch (cache_level) {
 310         case I915_CACHE_LLC_MLC:
 311                 if (INTEL_INFO(dev)->gen >= 6)
 312                         return AGP_USER_CACHED_MEMORY_LLC_MLC;
 313                 /* Older chipsets do not have this extra level of CPU
 314                  * cacheing, so fallthrough and request the PTE simply
 315                  * as cached.
 316                  */
 317         case I915_CACHE_LLC:
 318                 return AGP_USER_CACHED_MEMORY;
 319         default:
 320         case I915_CACHE_NONE:
 321                 return AGP_USER_MEMORY;
 322         }
 323 }
 324
 325 static bool do_idling(struct drm_i915_private *dev_priv)
 326 {
 327         bool ret = dev_priv->mm.interruptible;
 328
 329         if (unlikely(dev_priv->mm.gtt->do_idle_maps)) {
 330                 dev_priv->mm.interruptible = false;
 331                 if (i915_gpu_idle(dev_priv->dev, false)) {
 332                         DRM_ERROR("Couldn't idle GPU\n");
 333                         /* Wait a bit, in hopes it avoids the hang */
 334                         udelay(10);
 335                 }
 336         }
 337
 338         return ret;
 339 }
 340
 341 static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
 342 {
 343         if (unlikely(dev_priv->mm.gtt->do_idle_maps))
 344                 dev_priv->mm.interruptible = interruptible;
 345 }
 346
 347 void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 348 {
 349         struct drm_i915_private *dev_priv = dev->dev_private;
 350         struct drm_i915_gem_object *obj;
 351
 352         /* First fill our portion of the GTT with scratch pages */
 353         intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE,
 354                               (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE);
 355
 356         list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
 357                 i915_gem_clflush_object(obj);
 358                 i915_gem_gtt_rebind_object(obj, obj->cache_level);
 359         }
 360
 361         intel_gtt_chipset_flush();
 362 }
 363
 364 int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj)
 365 {
 366         struct drm_device *dev = obj->base.dev;
 367         struct drm_i915_private *dev_priv = dev->dev_private;
 368         unsigned int agp_type = cache_level_to_agp_type(dev, obj->cache_level);
 369         int ret;
 370
 371         if (dev_priv->mm.gtt->needs_dmar) {
 372                 ret = intel_gtt_map_memory(obj->pages,
 373                                            obj->base.size >> PAGE_SHIFT,
 374                                            &obj->sg_list,
 375                                            &obj->num_sg);
 376                 if (ret != 0)
 377                         return ret;
 378
 379                 intel_gtt_insert_sg_entries(obj->sg_list,
 380                                             obj->num_sg,
 381                                             obj->gtt_space->start >> PAGE_SHIFT,
 382                                             agp_type);
 383         } else
 384                 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
 385                                        obj->base.size >> PAGE_SHIFT,
 386                                        obj->pages,
 387                                        agp_type);
 388
 389         return 0;
 390 }
 391
 392 void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
 393                                 enum i915_cache_level cache_level)
 394 {
 395         struct drm_device *dev = obj->base.dev;
 396         struct drm_i915_private *dev_priv = dev->dev_private;
 397         unsigned int agp_type = cache_level_to_agp_type(dev, cache_level);
 398
 399         if (dev_priv->mm.gtt->needs_dmar) {
 400                 BUG_ON(!obj->sg_list);
 401
 402                 intel_gtt_insert_sg_entries(obj->sg_list,
 403                                             obj->num_sg,
 404                                             obj->gtt_space->start >> PAGE_SHIFT,
 405                                             agp_type);
 406         } else
 407                 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
 408                                        obj->base.size >> PAGE_SHIFT,
 409                                        obj->pages,
 410                                        agp_type);
 411 }
 412
 413 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 414 {
 415         struct drm_device *dev = obj->base.dev;
 416         struct drm_i915_private *dev_priv = dev->dev_private;
 417         bool interruptible;
 418
 419         interruptible = do_idling(dev_priv);
 420
 421         intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
 422                               obj->base.size >> PAGE_SHIFT);
 423
 424         if (obj->sg_list) {
 425                 intel_gtt_unmap_memory(obj->sg_list, obj->num_sg);
 426                 obj->sg_list = NULL;
 427         }
 428
 429         undo_idling(dev_priv, interruptible);
 430 }