drivers/char/agp/intel-gtt.c

   1 /*
   2  * Intel GTT (Graphics Translation Table) routines
   3  *
   4  * Caveat: This driver implements the linux agp interface, but this is far from
   5  * a agp driver! GTT support ended up here for purely historical reasons: The
   6  * old userspace intel graphics drivers needed an interface to map memory into
   7  * the GTT. And the drm provides a default interface for graphic devices sitting
   8  * on an agp port. So it made sense to fake the GTT support as an agp port to
   9  * avoid having to create a new api.
  10  *
  11  * With gem this does not make much sense anymore, just needlessly complicates
  12  * the code. But as long as the old graphics stack is still support, it's stuck
  13  * here.
  14  *
  15  * /fairy-tale-mode off
  16  */
  17
  18 #include <linux/module.h>
  19 #include <linux/pci.h>
  20 #include <linux/init.h>
  21 #include <linux/kernel.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/agp_backend.h>
  24 #include <asm/smp.h>
  25 #include "agp.h"
  26 #include "intel-agp.h"
  27 #include <linux/intel-gtt.h>
  28 #include <drm/intel-gtt.h>
  29
  30 /*
  31  * If we have Intel graphics, we're not going to have anything other than
  32  * an Intel IOMMU. So make the correct use of the PCI DMA API contingent
  33  * on the Intel IOMMU support (CONFIG_DMAR).
  34  * Only newer chipsets need to bother with this, of course.
  35  */
  36 #ifdef CONFIG_DMAR
  37 #define USE_PCI_DMA_API 1
  38 #else
  39 #define USE_PCI_DMA_API 0
  40 #endif
  41
  42 /* Max amount of stolen space, anything above will be returned to Linux */
  43 int intel_max_stolen = 32 * 1024 * 1024;
  44 EXPORT_SYMBOL(intel_max_stolen);
  45
  46 static const struct aper_size_info_fixed intel_i810_sizes[] =
  47 {
  48         {64, 16384, 4},
  49         /* The 32M mode still requires a 64k gatt */
  50         {32, 8192, 4}
  51 };
  52
  53 #define AGP_DCACHE_MEMORY       1
  54 #define AGP_PHYS_MEMORY         2
  55 #define INTEL_AGP_CACHED_MEMORY 3
  56
  57 static struct gatt_mask intel_i810_masks[] =
  58 {
  59         {.mask = I810_PTE_VALID, .type = 0},
  60         {.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY},
  61         {.mask = I810_PTE_VALID, .type = 0},
  62         {.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED,
  63          .type = INTEL_AGP_CACHED_MEMORY}
  64 };
  65
  66 #define INTEL_AGP_UNCACHED_MEMORY              0
  67 #define INTEL_AGP_CACHED_MEMORY_LLC            1
  68 #define INTEL_AGP_CACHED_MEMORY_LLC_GFDT       2
  69 #define INTEL_AGP_CACHED_MEMORY_LLC_MLC        3
  70 #define INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT   4
  71
  72 struct intel_gtt_driver {
  73         unsigned int gen : 8;
  74         unsigned int is_g33 : 1;
  75         unsigned int is_pineview : 1;
  76         unsigned int is_ironlake : 1;
  77         /* Chipset specific GTT setup */
  78         int (*setup)(void);
  79         void (*write_entry)(dma_addr_t addr, unsigned int entry, unsigned int flags);
  80         /* Flags is a more or less chipset specific opaque value.
  81          * For chipsets that need to support old ums (non-gem) code, this
  82          * needs to be identical to the various supported agp memory types! */
  83         bool (*check_flags)(unsigned int flags);
  84 };
  85
  86 static struct _intel_private {
  87         struct intel_gtt base;
  88         const struct intel_gtt_driver *driver;
  89         struct pci_dev *pcidev; /* device one */
  90         struct pci_dev *bridge_dev;
  91         u8 __iomem *registers;
  92         phys_addr_t gtt_bus_addr;
  93         phys_addr_t gma_bus_addr;
  94         phys_addr_t pte_bus_addr;
  95         u32 __iomem *gtt;               /* I915G */
  96         int num_dcache_entries;
  97         union {
  98                 void __iomem *i9xx_flush_page;
  99                 void *i8xx_flush_page;
 100         };
 101         struct page *i8xx_page;
 102         struct resource ifp_resource;
 103         int resource_valid;
 104         struct page *scratch_page;
 105         dma_addr_t scratch_page_dma;
 106 } intel_private;
 107
 108 #define INTEL_GTT_GEN   intel_private.driver->gen
 109 #define IS_G33          intel_private.driver->is_g33
 110 #define IS_PINEVIEW     intel_private.driver->is_pineview
 111 #define IS_IRONLAKE     intel_private.driver->is_ironlake
 112
 113 static void intel_agp_free_sglist(struct agp_memory *mem)
 114 {
 115         struct sg_table st;
 116
 117         st.sgl = mem->sg_list;
 118         st.orig_nents = st.nents = mem->page_count;
 119
 120         sg_free_table(&st);
 121
 122         mem->sg_list = NULL;
 123         mem->num_sg = 0;
 124 }
 125
 126 static int intel_agp_map_memory(struct agp_memory *mem)
 127 {
 128         struct sg_table st;
 129         struct scatterlist *sg;
 130         int i;
 131
 132         if (mem->sg_list)
 133                 return 0; /* already mapped (for e.g. resume */
 134
 135         DBG("try mapping %lu pages\n", (unsigned long)mem->page_count);
 136
 137         if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL))
 138                 goto err;
 139
 140         mem->sg_list = sg = st.sgl;
 141
 142         for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg))
 143                 sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0);
 144
 145         mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list,
 146                                  mem->page_count, PCI_DMA_BIDIRECTIONAL);
 147         if (unlikely(!mem->num_sg))
 148                 goto err;
 149
 150         return 0;
 151
 152 err:
 153         sg_free_table(&st);
 154         return -ENOMEM;
 155 }
 156
 157 static void intel_agp_unmap_memory(struct agp_memory *mem)
 158 {
 159         DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
 160
 161         pci_unmap_sg(intel_private.pcidev, mem->sg_list,
 162                      mem->page_count, PCI_DMA_BIDIRECTIONAL);
 163         intel_agp_free_sglist(mem);
 164 }
 165
 166 static int intel_i810_fetch_size(void)
 167 {
 168         u32 smram_miscc;
 169         struct aper_size_info_fixed *values;
 170
 171         pci_read_config_dword(intel_private.bridge_dev,
 172                               I810_SMRAM_MISCC, &smram_miscc);
 173         values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
 174
 175         if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) {
 176                 dev_warn(&intel_private.bridge_dev->dev, "i810 is disabled\n");
 177                 return 0;
 178         }
 179         if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) {
 180                 agp_bridge->current_size = (void *) (values + 1);
 181                 agp_bridge->aperture_size_idx = 1;
 182                 return values[1].size;
 183         } else {
 184                 agp_bridge->current_size = (void *) (values);
 185                 agp_bridge->aperture_size_idx = 0;
 186                 return values[0].size;
 187         }
 188
 189         return 0;
 190 }
 191
 192 static int intel_i810_configure(void)
 193 {
 194         struct aper_size_info_fixed *current_size;
 195         u32 temp;
 196         int i;
 197
 198         current_size = A_SIZE_FIX(agp_bridge->current_size);
 199
 200         if (!intel_private.registers) {
 201                 pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp);
 202                 temp &= 0xfff80000;
 203
 204                 intel_private.registers = ioremap(temp, 128 * 4096);
 205                 if (!intel_private.registers) {
 206                         dev_err(&intel_private.pcidev->dev,
 207                                 "can't remap memory\n");
 208                         return -ENOMEM;
 209                 }
 210         }
 211
 212         if ((readl(intel_private.registers+I810_DRAM_CTL)
 213                 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
 214                 /* This will need to be dynamically assigned */
 215                 dev_info(&intel_private.pcidev->dev,
 216                          "detected 4MB dedicated video ram\n");
 217                 intel_private.num_dcache_entries = 1024;
 218         }
 219         pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
 220         agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
 221         writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
 222         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
 223
 224         if (agp_bridge->driver->needs_scratch_page) {
 225                 for (i = 0; i < current_size->num_entries; i++) {
 226                         writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 227                 }
 228                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); /* PCI posting. */
 229         }
 230         global_cache_flush();
 231         return 0;
 232 }
 233
 234 static void intel_i810_cleanup(void)
 235 {
 236         writel(0, intel_private.registers+I810_PGETBL_CTL);
 237         readl(intel_private.registers); /* PCI Posting. */
 238         iounmap(intel_private.registers);
 239 }
 240
 241 static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
 242 {
 243         return;
 244 }
 245
 246 /* Exists to support ARGB cursors */
 247 static struct page *i8xx_alloc_pages(void)
 248 {
 249         struct page *page;
 250
 251         page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2);
 252         if (page == NULL)
 253                 return NULL;
 254
 255         if (set_pages_uc(page, 4) < 0) {
 256                 set_pages_wb(page, 4);
 257                 __free_pages(page, 2);
 258                 return NULL;
 259         }
 260         get_page(page);
 261         atomic_inc(&agp_bridge->current_memory_agp);
 262         return page;
 263 }
 264
 265 static void i8xx_destroy_pages(struct page *page)
 266 {
 267         if (page == NULL)
 268                 return;
 269
 270         set_pages_wb(page, 4);
 271         put_page(page);
 272         __free_pages(page, 2);
 273         atomic_dec(&agp_bridge->current_memory_agp);
 274 }
 275
 276 static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start,
 277                                 int type)
 278 {
 279         int i, j, num_entries;
 280         void *temp;
 281         int ret = -EINVAL;
 282         int mask_type;
 283
 284         if (mem->page_count == 0)
 285                 goto out;
 286
 287         temp = agp_bridge->current_size;
 288         num_entries = A_SIZE_FIX(temp)->num_entries;
 289
 290         if ((pg_start + mem->page_count) > num_entries)
 291                 goto out_err;
 292
 293
 294         for (j = pg_start; j < (pg_start + mem->page_count); j++) {
 295                 if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+j))) {
 296                         ret = -EBUSY;
 297                         goto out_err;
 298                 }
 299         }
 300
 301         if (type != mem->type)
 302                 goto out_err;
 303
 304         mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);
 305
 306         switch (mask_type) {
 307         case AGP_DCACHE_MEMORY:
 308                 if (!mem->is_flushed)
 309                         global_cache_flush();
 310                 for (i = pg_start; i < (pg_start + mem->page_count); i++) {
 311                         writel((i*4096)|I810_PTE_LOCAL|I810_PTE_VALID,
 312                                intel_private.registers+I810_PTE_BASE+(i*4));
 313                 }
 314                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
 315                 break;
 316         case AGP_PHYS_MEMORY:
 317         case AGP_NORMAL_MEMORY:
 318                 if (!mem->is_flushed)
 319                         global_cache_flush();
 320                 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
 321                         writel(agp_bridge->driver->mask_memory(agp_bridge,
 322                                         page_to_phys(mem->pages[i]), mask_type),
 323                                intel_private.registers+I810_PTE_BASE+(j*4));
 324                 }
 325                 readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
 326                 break;
 327         default:
 328                 goto out_err;
 329         }
 330
 331 out:
 332         ret = 0;
 333 out_err:
 334         mem->is_flushed = true;
 335         return ret;
 336 }
 337
 338 static int intel_i810_remove_entries(struct agp_memory *mem, off_t pg_start,
 339                                 int type)
 340 {
 341         int i;
 342
 343         if (mem->page_count == 0)
 344                 return 0;
 345
 346         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
 347                 writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 348         }
 349         readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
 350
 351         return 0;
 352 }
 353
 354 /*
 355  * The i810/i830 requires a physical address to program its mouse
 356  * pointer into hardware.
 357  * However the Xserver still writes to it through the agp aperture.
 358  */
 359 static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
 360 {
 361         struct agp_memory *new;
 362         struct page *page;
 363
 364         switch (pg_count) {
 365         case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge);
 366                 break;
 367         case 4:
 368                 /* kludge to get 4 physical pages for ARGB cursor */
 369                 page = i8xx_alloc_pages();
 370                 break;
 371         default:
 372                 return NULL;
 373         }
 374
 375         if (page == NULL)
 376                 return NULL;
 377
 378         new = agp_create_memory(pg_count);
 379         if (new == NULL)
 380                 return NULL;
 381
 382         new->pages[0] = page;
 383         if (pg_count == 4) {
 384                 /* kludge to get 4 physical pages for ARGB cursor */
 385                 new->pages[1] = new->pages[0] + 1;
 386                 new->pages[2] = new->pages[1] + 1;
 387                 new->pages[3] = new->pages[2] + 1;
 388         }
 389         new->page_count = pg_count;
 390         new->num_scratch_pages = pg_count;
 391         new->type = AGP_PHYS_MEMORY;
 392         new->physical = page_to_phys(new->pages[0]);
 393         return new;
 394 }
 395
 396 static struct agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type)
 397 {
 398         struct agp_memory *new;
 399
 400         if (type == AGP_DCACHE_MEMORY) {
 401                 if (pg_count != intel_private.num_dcache_entries)
 402                         return NULL;
 403
 404                 new = agp_create_memory(1);
 405                 if (new == NULL)
 406                         return NULL;
 407
 408                 new->type = AGP_DCACHE_MEMORY;
 409                 new->page_count = pg_count;
 410                 new->num_scratch_pages = 0;
 411                 agp_free_page_array(new);
 412                 return new;
 413         }
 414         if (type == AGP_PHYS_MEMORY)
 415                 return alloc_agpphysmem_i8xx(pg_count, type);
 416         return NULL;
 417 }
 418
 419 static void intel_i810_free_by_type(struct agp_memory *curr)
 420 {
 421         agp_free_key(curr->key);
 422         if (curr->type == AGP_PHYS_MEMORY) {
 423                 if (curr->page_count == 4)
 424                         i8xx_destroy_pages(curr->pages[0]);
 425                 else {
 426                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 427                                                              AGP_PAGE_DESTROY_UNMAP);
 428                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 429                                                              AGP_PAGE_DESTROY_FREE);
 430                 }
 431                 agp_free_page_array(curr);
 432         }
 433         kfree(curr);
 434 }
 435
 436 static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
 437                                             dma_addr_t addr, int type)
 438 {
 439         /* Type checking must be done elsewhere */
 440         return addr | bridge->driver->masks[type].mask;
 441 }
 442
 443 static int intel_gtt_setup_scratch_page(void)
 444 {
 445         struct page *page;
 446         dma_addr_t dma_addr;
 447
 448         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
 449         if (page == NULL)
 450                 return -ENOMEM;
 451         get_page(page);
 452         set_pages_uc(page, 1);
 453
 454         if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
 455                 dma_addr = pci_map_page(intel_private.pcidev, page, 0,
 456                                     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 457                 if (pci_dma_mapping_error(intel_private.pcidev, dma_addr))
 458                         return -EINVAL;
 459
 460                 intel_private.scratch_page_dma = dma_addr;
 461         } else
 462                 intel_private.scratch_page_dma = page_to_phys(page);
 463
 464         intel_private.scratch_page = page;
 465
 466         return 0;
 467 }
 468
 469 static const struct aper_size_info_fixed const intel_fake_agp_sizes[] = {
 470         {128, 32768, 5},
 471         /* The 64M mode still requires a 128k gatt */
 472         {64, 16384, 5},
 473         {256, 65536, 6},
 474         {512, 131072, 7},
 475 };
 476
 477 static unsigned int intel_gtt_stolen_entries(void)
 478 {
 479         u16 gmch_ctrl;
 480         u8 rdct;
 481         int local = 0;
 482         static const int ddt[4] = { 0, 16, 32, 64 };
 483         unsigned int overhead_entries, stolen_entries;
 484         unsigned int stolen_size = 0;
 485
 486         pci_read_config_word(intel_private.bridge_dev,
 487                              I830_GMCH_CTRL, &gmch_ctrl);
 488
 489         if (INTEL_GTT_GEN > 4 || IS_PINEVIEW)
 490                 overhead_entries = 0;
 491         else
 492                 overhead_entries = intel_private.base.gtt_mappable_entries
 493                         / 1024;
 494
 495         overhead_entries += 1; /* BIOS popup */
 496
 497         if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
 498             intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
 499                 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
 500                 case I830_GMCH_GMS_STOLEN_512:
 501                         stolen_size = KB(512);
 502                         break;
 503                 case I830_GMCH_GMS_STOLEN_1024:
 504                         stolen_size = MB(1);
 505                         break;
 506                 case I830_GMCH_GMS_STOLEN_8192:
 507                         stolen_size = MB(8);
 508                         break;
 509                 case I830_GMCH_GMS_LOCAL:
 510                         rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE);
 511                         stolen_size = (I830_RDRAM_ND(rdct) + 1) *
 512                                         MB(ddt[I830_RDRAM_DDT(rdct)]);
 513                         local = 1;
 514                         break;
 515                 default:
 516                         stolen_size = 0;
 517                         break;
 518                 }
 519         } else if (INTEL_GTT_GEN == 6) {
 520                 /*
 521                  * SandyBridge has new memory control reg at 0x50.w
 522                  */
 523                 u16 snb_gmch_ctl;
 524                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 525                 switch (snb_gmch_ctl & SNB_GMCH_GMS_STOLEN_MASK) {
 526                 case SNB_GMCH_GMS_STOLEN_32M:
 527                         stolen_size = MB(32);
 528                         break;
 529                 case SNB_GMCH_GMS_STOLEN_64M:
 530                         stolen_size = MB(64);
 531                         break;
 532                 case SNB_GMCH_GMS_STOLEN_96M:
 533                         stolen_size = MB(96);
 534                         break;
 535                 case SNB_GMCH_GMS_STOLEN_128M:
 536                         stolen_size = MB(128);
 537                         break;
 538                 case SNB_GMCH_GMS_STOLEN_160M:
 539                         stolen_size = MB(160);
 540                         break;
 541                 case SNB_GMCH_GMS_STOLEN_192M:
 542                         stolen_size = MB(192);
 543                         break;
 544                 case SNB_GMCH_GMS_STOLEN_224M:
 545                         stolen_size = MB(224);
 546                         break;
 547                 case SNB_GMCH_GMS_STOLEN_256M:
 548                         stolen_size = MB(256);
 549                         break;
 550                 case SNB_GMCH_GMS_STOLEN_288M:
 551                         stolen_size = MB(288);
 552                         break;
 553                 case SNB_GMCH_GMS_STOLEN_320M:
 554                         stolen_size = MB(320);
 555                         break;
 556                 case SNB_GMCH_GMS_STOLEN_352M:
 557                         stolen_size = MB(352);
 558                         break;
 559                 case SNB_GMCH_GMS_STOLEN_384M:
 560                         stolen_size = MB(384);
 561                         break;
 562                 case SNB_GMCH_GMS_STOLEN_416M:
 563                         stolen_size = MB(416);
 564                         break;
 565                 case SNB_GMCH_GMS_STOLEN_448M:
 566                         stolen_size = MB(448);
 567                         break;
 568                 case SNB_GMCH_GMS_STOLEN_480M:
 569                         stolen_size = MB(480);
 570                         break;
 571                 case SNB_GMCH_GMS_STOLEN_512M:
 572                         stolen_size = MB(512);
 573                         break;
 574                 }
 575         } else {
 576                 switch (gmch_ctrl & I855_GMCH_GMS_MASK) {
 577                 case I855_GMCH_GMS_STOLEN_1M:
 578                         stolen_size = MB(1);
 579                         break;
 580                 case I855_GMCH_GMS_STOLEN_4M:
 581                         stolen_size = MB(4);
 582                         break;
 583                 case I855_GMCH_GMS_STOLEN_8M:
 584                         stolen_size = MB(8);
 585                         break;
 586                 case I855_GMCH_GMS_STOLEN_16M:
 587                         stolen_size = MB(16);
 588                         break;
 589                 case I855_GMCH_GMS_STOLEN_32M:
 590                         stolen_size = MB(32);
 591                         break;
 592                 case I915_GMCH_GMS_STOLEN_48M:
 593                         stolen_size = MB(48);
 594                         break;
 595                 case I915_GMCH_GMS_STOLEN_64M:
 596                         stolen_size = MB(64);
 597                         break;
 598                 case G33_GMCH_GMS_STOLEN_128M:
 599                         stolen_size = MB(128);
 600                         break;
 601                 case G33_GMCH_GMS_STOLEN_256M:
 602                         stolen_size = MB(256);
 603                         break;
 604                 case INTEL_GMCH_GMS_STOLEN_96M:
 605                         stolen_size = MB(96);
 606                         break;
 607                 case INTEL_GMCH_GMS_STOLEN_160M:
 608                         stolen_size = MB(160);
 609                         break;
 610                 case INTEL_GMCH_GMS_STOLEN_224M:
 611                         stolen_size = MB(224);
 612                         break;
 613                 case INTEL_GMCH_GMS_STOLEN_352M:
 614                         stolen_size = MB(352);
 615                         break;
 616                 default:
 617                         stolen_size = 0;
 618                         break;
 619                 }
 620         }
 621
 622         if (!local && stolen_size > intel_max_stolen) {
 623                 dev_info(&intel_private.bridge_dev->dev,
 624                          "detected %dK stolen memory, trimming to %dK\n",
 625                          stolen_size / KB(1), intel_max_stolen / KB(1));
 626                 stolen_size = intel_max_stolen;
 627         } else if (stolen_size > 0) {
 628                 dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
 629                        stolen_size / KB(1), local ? "local" : "stolen");
 630         } else {
 631                 dev_info(&intel_private.bridge_dev->dev,
 632                        "no pre-allocated video memory detected\n");
 633                 stolen_size = 0;
 634         }
 635
 636         stolen_entries = stolen_size/KB(4) - overhead_entries;
 637
 638         return stolen_entries;
 639 }
 640
 641 static unsigned int intel_gtt_total_entries(void)
 642 {
 643         int size;
 644
 645         if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) {
 646                 u32 pgetbl_ctl;
 647                 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
 648
 649                 switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
 650                 case I965_PGETBL_SIZE_128KB:
 651                         size = KB(128);
 652                         break;
 653                 case I965_PGETBL_SIZE_256KB:
 654                         size = KB(256);
 655                         break;
 656                 case I965_PGETBL_SIZE_512KB:
 657                         size = KB(512);
 658                         break;
 659                 case I965_PGETBL_SIZE_1MB:
 660                         size = KB(1024);
 661                         break;
 662                 case I965_PGETBL_SIZE_2MB:
 663                         size = KB(2048);
 664                         break;
 665                 case I965_PGETBL_SIZE_1_5MB:
 666                         size = KB(1024 + 512);
 667                         break;
 668                 default:
 669                         dev_info(&intel_private.pcidev->dev,
 670                                  "unknown page table size, assuming 512KB\n");
 671                         size = KB(512);
 672                 }
 673
 674                 return size/4;
 675         } else if (INTEL_GTT_GEN == 6) {
 676                 u16 snb_gmch_ctl;
 677
 678                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 679                 switch (snb_gmch_ctl & SNB_GTT_SIZE_MASK) {
 680                 default:
 681                 case SNB_GTT_SIZE_0M:
 682                         printk(KERN_ERR "Bad GTT size mask: 0x%04x.\n", snb_gmch_ctl);
 683                         size = MB(0);
 684                         break;
 685                 case SNB_GTT_SIZE_1M:
 686                         size = MB(1);
 687                         break;
 688                 case SNB_GTT_SIZE_2M:
 689                         size = MB(2);
 690                         break;
 691                 }
 692                 return size/4;
 693         } else {
 694                 /* On previous hardware, the GTT size was just what was
 695                  * required to map the aperture.
 696                  */
 697                 return intel_private.base.gtt_mappable_entries;
 698         }
 699 }
 700
 701 static unsigned int intel_gtt_mappable_entries(void)
 702 {
 703         unsigned int aperture_size;
 704
 705         if (INTEL_GTT_GEN == 2) {
 706                 u16 gmch_ctrl;
 707
 708                 pci_read_config_word(intel_private.bridge_dev,
 709                                      I830_GMCH_CTRL, &gmch_ctrl);
 710
 711                 if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_64M)
 712                         aperture_size = MB(64);
 713                 else
 714                         aperture_size = MB(128);
 715         } else {
 716                 /* 9xx supports large sizes, just look at the length */
 717                 aperture_size = pci_resource_len(intel_private.pcidev, 2);
 718         }
 719
 720         return aperture_size >> PAGE_SHIFT;
 721 }
 722
 723 static void intel_gtt_teardown_scratch_page(void)
 724 {
 725         set_pages_wb(intel_private.scratch_page, 1);
 726         pci_unmap_page(intel_private.pcidev, intel_private.scratch_page_dma,
 727                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 728         put_page(intel_private.scratch_page);
 729         __free_page(intel_private.scratch_page);
 730 }
 731
 732 static void intel_gtt_cleanup(void)
 733 {
 734         if (intel_private.i9xx_flush_page)
 735                 iounmap(intel_private.i9xx_flush_page);
 736         if (intel_private.resource_valid)
 737                 release_resource(&intel_private.ifp_resource);
 738         intel_private.ifp_resource.start = 0;
 739         intel_private.resource_valid = 0;
 740         iounmap(intel_private.gtt);
 741         iounmap(intel_private.registers);
 742
 743         intel_gtt_teardown_scratch_page();
 744 }
 745
 746 static int intel_gtt_init(void)
 747 {
 748         u32 gtt_map_size;
 749         int ret;
 750
 751         ret = intel_private.driver->setup();
 752         if (ret != 0)
 753                 return ret;
 754
 755         intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries();
 756         intel_private.base.gtt_total_entries = intel_gtt_total_entries();
 757
 758         gtt_map_size = intel_private.base.gtt_total_entries * 4;
 759
 760         intel_private.gtt = ioremap(intel_private.gtt_bus_addr,
 761                                     gtt_map_size);
 762         if (!intel_private.gtt) {
 763                 iounmap(intel_private.registers);
 764                 return -ENOMEM;
 765         }
 766
 767         global_cache_flush();   /* FIXME: ? */
 768
 769         /* we have to call this as early as possible after the MMIO base address is known */
 770         intel_private.base.gtt_stolen_entries = intel_gtt_stolen_entries();
 771         if (intel_private.base.gtt_stolen_entries == 0) {
 772                 iounmap(intel_private.registers);
 773                 iounmap(intel_private.gtt);
 774                 return -ENOMEM;
 775         }
 776
 777         ret = intel_gtt_setup_scratch_page();
 778         if (ret != 0) {
 779                 intel_gtt_cleanup();
 780                 return ret;
 781         }
 782
 783         return 0;
 784 }
 785
 786 static int intel_fake_agp_fetch_size(void)
 787 {
 788         int num_sizes = ARRAY_SIZE(intel_fake_agp_sizes);
 789         unsigned int aper_size;
 790         int i;
 791
 792         aper_size = (intel_private.base.gtt_mappable_entries << PAGE_SHIFT)
 793                     / MB(1);
 794
 795         for (i = 0; i < num_sizes; i++) {
 796                 if (aper_size == intel_fake_agp_sizes[i].size) {
 797                         agp_bridge->current_size =
 798                                 (void *) (intel_fake_agp_sizes + i);
 799                         return aper_size;
 800                 }
 801         }
 802
 803         return 0;
 804 }
 805
 806 static void intel_i830_fini_flush(void)
 807 {
 808         kunmap(intel_private.i8xx_page);
 809         intel_private.i8xx_flush_page = NULL;
 810         unmap_page_from_agp(intel_private.i8xx_page);
 811
 812         __free_page(intel_private.i8xx_page);
 813         intel_private.i8xx_page = NULL;
 814 }
 815
 816 static void intel_i830_setup_flush(void)
 817 {
 818         /* return if we've already set the flush mechanism up */
 819         if (intel_private.i8xx_page)
 820                 return;
 821
 822         intel_private.i8xx_page = alloc_page(GFP_KERNEL | __GFP_ZERO | GFP_DMA32);
 823         if (!intel_private.i8xx_page)
 824                 return;
 825
 826         intel_private.i8xx_flush_page = kmap(intel_private.i8xx_page);
 827         if (!intel_private.i8xx_flush_page)
 828                 intel_i830_fini_flush();
 829 }
 830
 831 /* The chipset_flush interface needs to get data that has already been
 832  * flushed out of the CPU all the way out to main memory, because the GPU
 833  * doesn't snoop those buffers.
 834  *
 835  * The 8xx series doesn't have the same lovely interface for flushing the
 836  * chipset write buffers that the later chips do. According to the 865
 837  * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
 838  * that buffer out, we just fill 1KB and clflush it out, on the assumption
 839  * that it'll push whatever was in there out.  It appears to work.
 840  */
 841 static void intel_i830_chipset_flush(struct agp_bridge_data *bridge)
 842 {
 843         unsigned int *pg = intel_private.i8xx_flush_page;
 844
 845         memset(pg, 0, 1024);
 846
 847         if (cpu_has_clflush)
 848                 clflush_cache_range(pg, 1024);
 849         else if (wbinvd_on_all_cpus() != 0)
 850                 printk(KERN_ERR "Timed out waiting for cache flush.\n");
 851 }
 852
 853 static void i830_write_entry(dma_addr_t addr, unsigned int entry,
 854                              unsigned int flags)
 855 {
 856         u32 pte_flags = I810_PTE_VALID;
 857
 858         switch (flags) {
 859         case AGP_DCACHE_MEMORY:
 860                 pte_flags |= I810_PTE_LOCAL;
 861                 break;
 862         case AGP_USER_CACHED_MEMORY:
 863                 pte_flags |= I830_PTE_SYSTEM_CACHED;
 864                 break;
 865         }
 866
 867         writel(addr | pte_flags, intel_private.gtt + entry);
 868 }
 869
 870 static void intel_enable_gtt(void)
 871 {
 872         u32 gma_addr;
 873         u16 gmch_ctrl;
 874
 875         if (INTEL_GTT_GEN == 2)
 876                 pci_read_config_dword(intel_private.pcidev, I810_GMADDR,
 877                                       &gma_addr);
 878         else
 879                 pci_read_config_dword(intel_private.pcidev, I915_GMADDR,
 880                                       &gma_addr);
 881
 882         intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK);
 883
 884         pci_read_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, &gmch_ctrl);
 885         gmch_ctrl |= I830_GMCH_ENABLED;
 886         pci_write_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, gmch_ctrl);
 887
 888         writel(intel_private.pte_bus_addr|I810_PGETBL_ENABLED,
 889                intel_private.registers+I810_PGETBL_CTL);
 890         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
 891 }
 892
 893 static int i830_setup(void)
 894 {
 895         u32 reg_addr;
 896
 897         pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
 898         reg_addr &= 0xfff80000;
 899
 900         intel_private.registers = ioremap(reg_addr, KB(64));
 901         if (!intel_private.registers)
 902                 return -ENOMEM;
 903
 904         intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
 905         intel_private.pte_bus_addr =
 906                 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
 907
 908         intel_i830_setup_flush();
 909
 910         return 0;
 911 }
 912
 913 static int intel_fake_agp_create_gatt_table(struct agp_bridge_data *bridge)
 914 {
 915         agp_bridge->gatt_table_real = NULL;
 916         agp_bridge->gatt_table = NULL;
 917         agp_bridge->gatt_bus_addr = 0;
 918
 919         return 0;
 920 }
 921
 922 static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge)
 923 {
 924         return 0;
 925 }
 926
 927 static int intel_fake_agp_configure(void)
 928 {
 929         int i;
 930
 931         intel_enable_gtt();
 932
 933         agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;
 934
 935         for (i = intel_private.base.gtt_stolen_entries;
 936                         i < intel_private.base.gtt_total_entries; i++) {
 937                 intel_private.driver->write_entry(intel_private.scratch_page_dma,
 938                                                   i, 0);
 939         }
 940         readl(intel_private.gtt+i-1);   /* PCI Posting. */
 941
 942         global_cache_flush();
 943
 944         return 0;
 945 }
 946
 947 static bool i830_check_flags(unsigned int flags)
 948 {
 949         switch (flags) {
 950         case 0:
 951         case AGP_PHYS_MEMORY:
 952         case AGP_USER_CACHED_MEMORY:
 953         case AGP_USER_MEMORY:
 954                 return true;
 955         }
 956
 957         return false;
 958 }
 959
 960 static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
 961                                         unsigned int sg_len,
 962                                         unsigned int pg_start,
 963                                         unsigned int flags)
 964 {
 965         struct scatterlist *sg;
 966         unsigned int len, m;
 967         int i, j;
 968
 969         j = pg_start;
 970
 971         /* sg may merge pages, but we have to separate
 972          * per-page addr for GTT */
 973         for_each_sg(sg_list, sg, sg_len, i) {
 974                 len = sg_dma_len(sg) >> PAGE_SHIFT;
 975                 for (m = 0; m < len; m++) {
 976                         dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 977                         intel_private.driver->write_entry(addr,
 978                                                           j, flags);
 979                         j++;
 980                 }
 981         }
 982         readl(intel_private.gtt+j-1);
 983 }
 984
 985 static int intel_fake_agp_insert_entries(struct agp_memory *mem,
 986                                          off_t pg_start, int type)
 987 {
 988         int i, j;
 989         int ret = -EINVAL;
 990
 991         if (mem->page_count == 0)
 992                 goto out;
 993
 994         if (pg_start < intel_private.base.gtt_stolen_entries) {
 995                 dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
 996                            "pg_start == 0x%.8lx, gtt_stolen_entries == 0x%.8x\n",
 997                            pg_start, intel_private.base.gtt_stolen_entries);
 998
 999                 dev_info(&intel_private.pcidev->dev,
1000                          "trying to insert into local/stolen memory\n");
1001                 goto out_err;
1002         }
1003
1004         if ((pg_start + mem->page_count) > intel_private.base.gtt_total_entries)
1005                 goto out_err;
1006
1007         if (type != mem->type)
1008                 goto out_err;
1009
1010         if (!intel_private.driver->check_flags(type))
1011                 goto out_err;
1012
1013         if (!mem->is_flushed)
1014                 global_cache_flush();
1015
1016         if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
1017                 ret = intel_agp_map_memory(mem);
1018                 if (ret != 0)
1019                         return ret;
1020
1021                 intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg,
1022                                             pg_start, type);
1023         } else {
1024                 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
1025                         dma_addr_t addr = page_to_phys(mem->pages[i]);
1026                         intel_private.driver->write_entry(addr,
1027                                                           j, type);
1028                 }
1029                 readl(intel_private.gtt+j-1);
1030         }
1031
1032 out:
1033         ret = 0;
1034 out_err:
1035         mem->is_flushed = true;
1036         return ret;
1037 }
1038
1039 static int intel_fake_agp_remove_entries(struct agp_memory *mem,
1040                                          off_t pg_start, int type)
1041 {
1042         int i;
1043
1044         if (mem->page_count == 0)
1045                 return 0;
1046
1047         if (pg_start < intel_private.base.gtt_stolen_entries) {
1048                 dev_info(&intel_private.pcidev->dev,
1049                          "trying to disable local/stolen memory\n");
1050                 return -EINVAL;
1051         }
1052
1053         if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2)
1054                 intel_agp_unmap_memory(mem);
1055
1056         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
1057                 intel_private.driver->write_entry(intel_private.scratch_page_dma,
1058                                                   i, 0);
1059         }
1060         readl(intel_private.gtt+i-1);
1061
1062         return 0;
1063 }
1064
1065 static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count,
1066                                                        int type)
1067 {
1068         if (type == AGP_PHYS_MEMORY)
1069                 return alloc_agpphysmem_i8xx(pg_count, type);
1070         /* always return NULL for other allocation types for now */
1071         return NULL;
1072 }
1073
1074 static int intel_alloc_chipset_flush_resource(void)
1075 {
1076         int ret;
1077         ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE,
1078                                      PAGE_SIZE, PCIBIOS_MIN_MEM, 0,
1079                                      pcibios_align_resource, intel_private.bridge_dev);
1080
1081         return ret;
1082 }
1083
1084 static void intel_i915_setup_chipset_flush(void)
1085 {
1086         int ret;
1087         u32 temp;
1088
1089         pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp);
1090         if (!(temp & 0x1)) {
1091                 intel_alloc_chipset_flush_resource();
1092                 intel_private.resource_valid = 1;
1093                 pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1094         } else {
1095                 temp &= ~1;
1096
1097                 intel_private.resource_valid = 1;
1098                 intel_private.ifp_resource.start = temp;
1099                 intel_private.ifp_resource.end = temp + PAGE_SIZE;
1100                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1101                 /* some BIOSes reserve this area in a pnp some don't */
1102                 if (ret)
1103                         intel_private.resource_valid = 0;
1104         }
1105 }
1106
1107 static void intel_i965_g33_setup_chipset_flush(void)
1108 {
1109         u32 temp_hi, temp_lo;
1110         int ret;
1111
1112         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi);
1113         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo);
1114
1115         if (!(temp_lo & 0x1)) {
1116
1117                 intel_alloc_chipset_flush_resource();
1118
1119                 intel_private.resource_valid = 1;
1120                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4,
1121                         upper_32_bits(intel_private.ifp_resource.start));
1122                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1123         } else {
1124                 u64 l64;
1125
1126                 temp_lo &= ~0x1;
1127                 l64 = ((u64)temp_hi << 32) | temp_lo;
1128
1129                 intel_private.resource_valid = 1;
1130                 intel_private.ifp_resource.start = l64;
1131                 intel_private.ifp_resource.end = l64 + PAGE_SIZE;
1132                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1133                 /* some BIOSes reserve this area in a pnp some don't */
1134                 if (ret)
1135                         intel_private.resource_valid = 0;
1136         }
1137 }
1138
1139 static void intel_i9xx_setup_flush(void)
1140 {
1141         /* return if already configured */
1142         if (intel_private.ifp_resource.start)
1143                 return;
1144
1145         if (INTEL_GTT_GEN == 6)
1146                 return;
1147
1148         /* setup a resource for this object */
1149         intel_private.ifp_resource.name = "Intel Flush Page";
1150         intel_private.ifp_resource.flags = IORESOURCE_MEM;
1151
1152         /* Setup chipset flush for 915 */
1153         if (IS_G33 || INTEL_GTT_GEN >= 4) {
1154                 intel_i965_g33_setup_chipset_flush();
1155         } else {
1156                 intel_i915_setup_chipset_flush();
1157         }
1158
1159         if (intel_private.ifp_resource.start)
1160                 intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE);
1161         if (!intel_private.i9xx_flush_page)
1162                 dev_err(&intel_private.pcidev->dev,
1163                         "can't ioremap flush page - no chipset flushing\n");
1164 }
1165
1166 static void intel_i915_chipset_flush(struct agp_bridge_data *bridge)
1167 {
1168         if (intel_private.i9xx_flush_page)
1169                 writel(1, intel_private.i9xx_flush_page);
1170 }
1171
1172 static void i965_write_entry(dma_addr_t addr, unsigned int entry,
1173                              unsigned int flags)
1174 {
1175         /* Shift high bits down */
1176         addr |= (addr >> 28) & 0xf0;
1177         writel(addr | I810_PTE_VALID, intel_private.gtt + entry);
1178 }
1179
1180 static bool gen6_check_flags(unsigned int flags)
1181 {
1182         return true;
1183 }
1184
1185 static void gen6_write_entry(dma_addr_t addr, unsigned int entry,
1186                              unsigned int flags)
1187 {
1188         unsigned int type_mask = flags & ~AGP_USER_CACHED_MEMORY_GFDT;
1189         unsigned int gfdt = flags & AGP_USER_CACHED_MEMORY_GFDT;
1190         u32 pte_flags;
1191
1192         if (type_mask == AGP_USER_UNCACHED_MEMORY)
1193                 pte_flags = GEN6_PTE_UNCACHED;
1194         else if (type_mask == AGP_USER_CACHED_MEMORY_LLC_MLC) {
1195                 pte_flags = GEN6_PTE_LLC;
1196                 if (gfdt)
1197                         pte_flags |= GEN6_PTE_GFDT;
1198         } else { /* set 'normal'/'cached' to LLC by default */
1199                 pte_flags = GEN6_PTE_LLC_MLC;
1200                 if (gfdt)
1201                         pte_flags |= GEN6_PTE_GFDT;
1202         }
1203
1204         /* gen6 has bit11-4 for physical addr bit39-32 */
1205         addr |= (addr >> 28) & 0xff0;
1206         writel(addr | pte_flags, intel_private.gtt + entry);
1207 }
1208
1209 static int i9xx_setup(void)
1210 {
1211         u32 reg_addr;
1212
1213         pci_read_config_dword(intel_private.pcidev, I915_MMADDR, &reg_addr);
1214
1215         reg_addr &= 0xfff80000;
1216
1217         intel_private.registers = ioremap(reg_addr, 128 * 4096);
1218         if (!intel_private.registers)
1219                 return -ENOMEM;
1220
1221         if (INTEL_GTT_GEN == 3) {
1222                 u32 gtt_addr;
1223
1224                 pci_read_config_dword(intel_private.pcidev,
1225                                       I915_PTEADDR, &gtt_addr);
1226                 intel_private.gtt_bus_addr = gtt_addr;
1227         } else {
1228                 u32 gtt_offset;
1229
1230                 switch (INTEL_GTT_GEN) {
1231                 case 5:
1232                 case 6:
1233                         gtt_offset = MB(2);
1234                         break;
1235                 case 4:
1236                 default:
1237                         gtt_offset =  KB(512);
1238                         break;
1239                 }
1240                 intel_private.gtt_bus_addr = reg_addr + gtt_offset;
1241         }
1242
1243         intel_private.pte_bus_addr =
1244                 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
1245
1246         intel_i9xx_setup_flush();
1247
1248         return 0;
1249 }
1250
1251 static const struct agp_bridge_driver intel_810_driver = {
1252         .owner                  = THIS_MODULE,
1253         .aperture_sizes         = intel_i810_sizes,
1254         .size_type              = FIXED_APER_SIZE,
1255         .num_aperture_sizes     = 2,
1256         .needs_scratch_page     = true,
1257         .configure              = intel_i810_configure,
1258         .fetch_size             = intel_i810_fetch_size,
1259         .cleanup                = intel_i810_cleanup,
1260         .mask_memory            = intel_i810_mask_memory,
1261         .masks                  = intel_i810_masks,
1262         .agp_enable             = intel_fake_agp_enable,
1263         .cache_flush            = global_cache_flush,
1264         .create_gatt_table      = agp_generic_create_gatt_table,
1265         .free_gatt_table        = agp_generic_free_gatt_table,
1266         .insert_memory          = intel_i810_insert_entries,
1267         .remove_memory          = intel_i810_remove_entries,
1268         .alloc_by_type          = intel_i810_alloc_by_type,
1269         .free_by_type           = intel_i810_free_by_type,
1270         .agp_alloc_page         = agp_generic_alloc_page,
1271         .agp_alloc_pages        = agp_generic_alloc_pages,
1272         .agp_destroy_page       = agp_generic_destroy_page,
1273         .agp_destroy_pages      = agp_generic_destroy_pages,
1274         .agp_type_to_mask_type  = agp_generic_type_to_mask_type,
1275 };
1276
1277 static const struct agp_bridge_driver intel_830_driver = {
1278         .owner                  = THIS_MODULE,
1279         .size_type              = FIXED_APER_SIZE,
1280         .aperture_sizes         = intel_fake_agp_sizes,
1281         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1282         .configure              = intel_fake_agp_configure,
1283         .fetch_size             = intel_fake_agp_fetch_size,
1284         .cleanup                = intel_gtt_cleanup,
1285         .agp_enable             = intel_fake_agp_enable,
1286         .cache_flush            = global_cache_flush,
1287         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1288         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1289         .insert_memory          = intel_fake_agp_insert_entries,
1290         .remove_memory          = intel_fake_agp_remove_entries,
1291         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1292         .free_by_type           = intel_i810_free_by_type,
1293         .agp_alloc_page         = agp_generic_alloc_page,
1294         .agp_alloc_pages        = agp_generic_alloc_pages,
1295         .agp_destroy_page       = agp_generic_destroy_page,
1296         .agp_destroy_pages      = agp_generic_destroy_pages,
1297         .chipset_flush          = intel_i830_chipset_flush,
1298 };
1299
1300 static const struct agp_bridge_driver intel_915_driver = {
1301         .owner                  = THIS_MODULE,
1302         .size_type              = FIXED_APER_SIZE,
1303         .aperture_sizes         = intel_fake_agp_sizes,
1304         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1305         .configure              = intel_fake_agp_configure,
1306         .fetch_size             = intel_fake_agp_fetch_size,
1307         .cleanup                = intel_gtt_cleanup,
1308         .agp_enable             = intel_fake_agp_enable,
1309         .cache_flush            = global_cache_flush,
1310         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1311         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1312         .insert_memory          = intel_fake_agp_insert_entries,
1313         .remove_memory          = intel_fake_agp_remove_entries,
1314         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1315         .free_by_type           = intel_i810_free_by_type,
1316         .agp_alloc_page         = agp_generic_alloc_page,
1317         .agp_alloc_pages        = agp_generic_alloc_pages,
1318         .agp_destroy_page       = agp_generic_destroy_page,
1319         .agp_destroy_pages      = agp_generic_destroy_pages,
1320         .chipset_flush          = intel_i915_chipset_flush,
1321 };
1322
1323 static const struct agp_bridge_driver intel_i965_driver = {
1324         .owner                  = THIS_MODULE,
1325         .size_type              = FIXED_APER_SIZE,
1326         .aperture_sizes         = intel_fake_agp_sizes,
1327         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1328         .configure              = intel_fake_agp_configure,
1329         .fetch_size             = intel_fake_agp_fetch_size,
1330         .cleanup                = intel_gtt_cleanup,
1331         .agp_enable             = intel_fake_agp_enable,
1332         .cache_flush            = global_cache_flush,
1333         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1334         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1335         .insert_memory          = intel_fake_agp_insert_entries,
1336         .remove_memory          = intel_fake_agp_remove_entries,
1337         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1338         .free_by_type           = intel_i810_free_by_type,
1339         .agp_alloc_page         = agp_generic_alloc_page,
1340         .agp_alloc_pages        = agp_generic_alloc_pages,
1341         .agp_destroy_page       = agp_generic_destroy_page,
1342         .agp_destroy_pages      = agp_generic_destroy_pages,
1343         .chipset_flush          = intel_i915_chipset_flush,
1344 };
1345
1346 static const struct agp_bridge_driver intel_gen6_driver = {
1347         .owner                  = THIS_MODULE,
1348         .size_type              = FIXED_APER_SIZE,
1349         .aperture_sizes         = intel_fake_agp_sizes,
1350         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1351         .configure              = intel_fake_agp_configure,
1352         .fetch_size             = intel_fake_agp_fetch_size,
1353         .cleanup                = intel_gtt_cleanup,
1354         .agp_enable             = intel_fake_agp_enable,
1355         .cache_flush            = global_cache_flush,
1356         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1357         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1358         .insert_memory          = intel_fake_agp_insert_entries,
1359         .remove_memory          = intel_fake_agp_remove_entries,
1360         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1361         .free_by_type           = intel_i810_free_by_type,
1362         .agp_alloc_page         = agp_generic_alloc_page,
1363         .agp_alloc_pages        = agp_generic_alloc_pages,
1364         .agp_destroy_page       = agp_generic_destroy_page,
1365         .agp_destroy_pages      = agp_generic_destroy_pages,
1366         .chipset_flush          = intel_i915_chipset_flush,
1367 };
1368
1369 static const struct agp_bridge_driver intel_g33_driver = {
1370         .owner                  = THIS_MODULE,
1371         .size_type              = FIXED_APER_SIZE,
1372         .aperture_sizes         = intel_fake_agp_sizes,
1373         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1374         .configure              = intel_fake_agp_configure,
1375         .fetch_size             = intel_fake_agp_fetch_size,
1376         .cleanup                = intel_gtt_cleanup,
1377         .agp_enable             = intel_fake_agp_enable,
1378         .cache_flush            = global_cache_flush,
1379         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1380         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1381         .insert_memory          = intel_fake_agp_insert_entries,
1382         .remove_memory          = intel_fake_agp_remove_entries,
1383         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1384         .free_by_type           = intel_i810_free_by_type,
1385         .agp_alloc_page         = agp_generic_alloc_page,
1386         .agp_alloc_pages        = agp_generic_alloc_pages,
1387         .agp_destroy_page       = agp_generic_destroy_page,
1388         .agp_destroy_pages      = agp_generic_destroy_pages,
1389         .chipset_flush          = intel_i915_chipset_flush,
1390 };
1391
1392 static const struct intel_gtt_driver i81x_gtt_driver = {
1393         .gen = 1,
1394 };
1395 static const struct intel_gtt_driver i8xx_gtt_driver = {
1396         .gen = 2,
1397         .setup = i830_setup,
1398         .write_entry = i830_write_entry,
1399         .check_flags = i830_check_flags,
1400 };
1401 static const struct intel_gtt_driver i915_gtt_driver = {
1402         .gen = 3,
1403         .setup = i9xx_setup,
1404         /* i945 is the last gpu to need phys mem (for overlay and cursors). */
1405         .write_entry = i830_write_entry,
1406         .check_flags = i830_check_flags,
1407 };
1408 static const struct intel_gtt_driver g33_gtt_driver = {
1409         .gen = 3,
1410         .is_g33 = 1,
1411         .setup = i9xx_setup,
1412         .write_entry = i965_write_entry,
1413         .check_flags = i830_check_flags,
1414 };
1415 static const struct intel_gtt_driver pineview_gtt_driver = {
1416         .gen = 3,
1417         .is_pineview = 1, .is_g33 = 1,
1418         .setup = i9xx_setup,
1419         .write_entry = i965_write_entry,
1420         .check_flags = i830_check_flags,
1421 };
1422 static const struct intel_gtt_driver i965_gtt_driver = {
1423         .gen = 4,
1424         .setup = i9xx_setup,
1425         .write_entry = i965_write_entry,
1426         .check_flags = i830_check_flags,
1427 };
1428 static const struct intel_gtt_driver g4x_gtt_driver = {
1429         .gen = 5,
1430         .setup = i9xx_setup,
1431         .write_entry = i965_write_entry,
1432         .check_flags = i830_check_flags,
1433 };
1434 static const struct intel_gtt_driver ironlake_gtt_driver = {
1435         .gen = 5,
1436         .is_ironlake = 1,
1437         .setup = i9xx_setup,
1438         .write_entry = i965_write_entry,
1439         .check_flags = i830_check_flags,
1440 };
1441 static const struct intel_gtt_driver sandybridge_gtt_driver = {
1442         .gen = 6,
1443         .setup = i9xx_setup,
1444         .write_entry = gen6_write_entry,
1445         .check_flags = gen6_check_flags,
1446 };
1447
1448 /* Table to describe Intel GMCH and AGP/PCIE GART drivers.  At least one of
1449  * driver and gmch_driver must be non-null, and find_gmch will determine
1450  * which one should be used if a gmch_chip_id is present.
1451  */
1452 static const struct intel_gtt_driver_description {
1453         unsigned int gmch_chip_id;
1454         char *name;
1455         const struct agp_bridge_driver *gmch_driver;
1456         const struct intel_gtt_driver *gtt_driver;
1457 } intel_gtt_chipsets[] = {
1458         { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &intel_810_driver,
1459                 &i81x_gtt_driver},
1460         { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &intel_810_driver,
1461                 &i81x_gtt_driver},
1462         { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &intel_810_driver,
1463                 &i81x_gtt_driver},
1464         { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &intel_810_driver,
1465                 &i81x_gtt_driver},
1466         { PCI_DEVICE_ID_INTEL_82830_CGC, "830M",
1467                 &intel_830_driver , &i8xx_gtt_driver},
1468         { PCI_DEVICE_ID_INTEL_82845G_IG, "830M",
1469                 &intel_830_driver , &i8xx_gtt_driver},
1470         { PCI_DEVICE_ID_INTEL_82854_IG, "854",
1471                 &intel_830_driver , &i8xx_gtt_driver},
1472         { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM",
1473                 &intel_830_driver , &i8xx_gtt_driver},
1474         { PCI_DEVICE_ID_INTEL_82865_IG, "865",
1475                 &intel_830_driver , &i8xx_gtt_driver},
1476         { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)",
1477                 &intel_915_driver , &i915_gtt_driver },
1478         { PCI_DEVICE_ID_INTEL_82915G_IG, "915G",
1479                 &intel_915_driver , &i915_gtt_driver },
1480         { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM",
1481                 &intel_915_driver , &i915_gtt_driver },
1482         { PCI_DEVICE_ID_INTEL_82945G_IG, "945G",
1483                 &intel_915_driver , &i915_gtt_driver },
1484         { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM",
1485                 &intel_915_driver , &i915_gtt_driver },
1486         { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME",
1487                 &intel_915_driver , &i915_gtt_driver },
1488         { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ",
1489                 &intel_i965_driver , &i965_gtt_driver },
1490         { PCI_DEVICE_ID_INTEL_82G35_IG, "G35",
1491                 &intel_i965_driver , &i965_gtt_driver },
1492         { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q",
1493                 &intel_i965_driver , &i965_gtt_driver },
1494         { PCI_DEVICE_ID_INTEL_82965G_IG, "965G",
1495                 &intel_i965_driver , &i965_gtt_driver },
1496         { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM",
1497                 &intel_i965_driver , &i965_gtt_driver },
1498         { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE",
1499                 &intel_i965_driver , &i965_gtt_driver },
1500         { PCI_DEVICE_ID_INTEL_G33_IG, "G33",
1501                 &intel_g33_driver , &g33_gtt_driver },
1502         { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35",
1503                 &intel_g33_driver , &g33_gtt_driver },
1504         { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33",
1505                 &intel_g33_driver , &g33_gtt_driver },
1506         { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150",
1507                 &intel_g33_driver , &pineview_gtt_driver },
1508         { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150",
1509                 &intel_g33_driver , &pineview_gtt_driver },
1510         { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45",
1511                 &intel_i965_driver , &g4x_gtt_driver },
1512         { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake",
1513                 &intel_i965_driver , &g4x_gtt_driver },
1514         { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43",
1515                 &intel_i965_driver , &g4x_gtt_driver },
1516         { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43",
1517                 &intel_i965_driver , &g4x_gtt_driver },
1518         { PCI_DEVICE_ID_INTEL_B43_IG, "B43",
1519                 &intel_i965_driver , &g4x_gtt_driver },
1520         { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43",
1521                 &intel_i965_driver , &g4x_gtt_driver },
1522         { PCI_DEVICE_ID_INTEL_G41_IG, "G41",
1523                 &intel_i965_driver , &g4x_gtt_driver },
1524         { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG,
1525             "HD Graphics", &intel_i965_driver , &ironlake_gtt_driver },
1526         { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG,
1527             "HD Graphics", &intel_i965_driver , &ironlake_gtt_driver },
1528         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG,
1529             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1530         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG,
1531             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1532         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG,
1533             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1534         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG,
1535             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1536         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG,
1537             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1538         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG,
1539             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1540         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG,
1541             "Sandybridge", &intel_gen6_driver , &sandybridge_gtt_driver },
1542         { 0, NULL, NULL }
1543 };
1544
1545 static int find_gmch(u16 device)
1546 {
1547         struct pci_dev *gmch_device;
1548
1549         gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
1550         if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) {
1551                 gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL,
1552                                              device, gmch_device);
1553         }
1554
1555         if (!gmch_device)
1556                 return 0;
1557
1558         intel_private.pcidev = gmch_device;
1559         return 1;
1560 }
1561
1562 int intel_gmch_probe(struct pci_dev *pdev,
1563                                       struct agp_bridge_data *bridge)
1564 {
1565         int i, mask;
1566         bridge->driver = NULL;
1567
1568         for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) {
1569                 if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) {
1570                         bridge->driver =
1571                                 intel_gtt_chipsets[i].gmch_driver;
1572                         intel_private.driver =
1573                                 intel_gtt_chipsets[i].gtt_driver;
1574                         break;
1575                 }
1576         }
1577
1578         if (!bridge->driver)
1579                 return 0;
1580
1581         bridge->dev_private_data = &intel_private;
1582         bridge->dev = pdev;
1583
1584         intel_private.bridge_dev = pci_dev_get(pdev);
1585
1586         dev_info(&pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name);
1587
1588         if (intel_private.driver->write_entry == gen6_write_entry)
1589                 mask = 40;
1590         else if (intel_private.driver->write_entry == i965_write_entry)
1591                 mask = 36;
1592         else
1593                 mask = 32;
1594
1595         if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask)))
1596                 dev_err(&intel_private.pcidev->dev,
1597                         "set gfx device dma mask %d-bit failed!\n", mask);
1598         else
1599                 pci_set_consistent_dma_mask(intel_private.pcidev,
1600                                             DMA_BIT_MASK(mask));
1601
1602         if (bridge->driver == &intel_810_driver)
1603                 return 1;
1604
1605         if (intel_gtt_init() != 0)
1606                 return 0;
1607
1608         return 1;
1609 }
1610 EXPORT_SYMBOL(intel_gmch_probe);
1611
1612 struct intel_gtt *intel_gtt_get(void)
1613 {
1614         return &intel_private.base;
1615 }
1616 EXPORT_SYMBOL(intel_gtt_get);
1617
1618 void intel_gmch_remove(struct pci_dev *pdev)
1619 {
1620         if (intel_private.pcidev)
1621                 pci_dev_put(intel_private.pcidev);
1622         if (intel_private.bridge_dev)
1623                 pci_dev_put(intel_private.bridge_dev);
1624 }
1625 EXPORT_SYMBOL(intel_gmch_remove);
1626
1627 MODULE_AUTHOR("Dave Jones <davej@redhat.com>");
1628 MODULE_LICENSE("GPL and additional rights");