drivers/char/agp/intel-gtt.c

   1 /*
   2  * Intel GTT (Graphics Translation Table) routines
   3  *
   4  * Caveat: This driver implements the linux agp interface, but this is far from
   5  * a agp driver! GTT support ended up here for purely historical reasons: The
   6  * old userspace intel graphics drivers needed an interface to map memory into
   7  * the GTT. And the drm provides a default interface for graphic devices sitting
   8  * on an agp port. So it made sense to fake the GTT support as an agp port to
   9  * avoid having to create a new api.
  10  *
  11  * With gem this does not make much sense anymore, just needlessly complicates
  12  * the code. But as long as the old graphics stack is still support, it's stuck
  13  * here.
  14  *
  15  * /fairy-tale-mode off
  16  */
  17
  18 #include <linux/module.h>
  19 #include <linux/pci.h>
  20 #include <linux/init.h>
  21 #include <linux/kernel.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/agp_backend.h>
  24 #include <asm/smp.h>
  25 #include "agp.h"
  26 #include "intel-agp.h"
  27 #include <linux/intel-gtt.h>
  28 #include <drm/intel-gtt.h>
  29
  30 /*
  31  * If we have Intel graphics, we're not going to have anything other than
  32  * an Intel IOMMU. So make the correct use of the PCI DMA API contingent
  33  * on the Intel IOMMU support (CONFIG_DMAR).
  34  * Only newer chipsets need to bother with this, of course.
  35  */
  36 #ifdef CONFIG_DMAR
  37 #define USE_PCI_DMA_API 1
  38 #else
  39 #define USE_PCI_DMA_API 0
  40 #endif
  41
  42 static const struct aper_size_info_fixed intel_i810_sizes[] =
  43 {
  44         {64, 16384, 4},
  45         /* The 32M mode still requires a 64k gatt */
  46         {32, 8192, 4}
  47 };
  48
  49 #define AGP_DCACHE_MEMORY       1
  50 #define AGP_PHYS_MEMORY         2
  51 #define INTEL_AGP_CACHED_MEMORY 3
  52
  53 static struct gatt_mask intel_i810_masks[] =
  54 {
  55         {.mask = I810_PTE_VALID, .type = 0},
  56         {.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY},
  57         {.mask = I810_PTE_VALID, .type = 0},
  58         {.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED,
  59          .type = INTEL_AGP_CACHED_MEMORY}
  60 };
  61
  62 struct intel_gtt_driver {
  63         unsigned int gen : 8;
  64         unsigned int is_g33 : 1;
  65         unsigned int is_pineview : 1;
  66         unsigned int is_ironlake : 1;
  67         unsigned int has_pgtbl_enable : 1;
  68         unsigned int dma_mask_size : 8;
  69         /* Chipset specific GTT setup */
  70         int (*setup)(void);
  71         /* This should undo anything done in ->setup() save the unmapping
  72          * of the mmio register file, that's done in the generic code. */
  73         void (*cleanup)(void);
  74         void (*write_entry)(dma_addr_t addr, unsigned int entry, unsigned int flags);
  75         /* Flags is a more or less chipset specific opaque value.
  76          * For chipsets that need to support old ums (non-gem) code, this
  77          * needs to be identical to the various supported agp memory types! */
  78         bool (*check_flags)(unsigned int flags);
  79         void (*chipset_flush)(void);
  80 };
  81
  82 static struct _intel_private {
  83         struct intel_gtt base;
  84         const struct intel_gtt_driver *driver;
  85         struct pci_dev *pcidev; /* device one */
  86         struct pci_dev *bridge_dev;
  87         u8 __iomem *registers;
  88         phys_addr_t gtt_bus_addr;
  89         phys_addr_t gma_bus_addr;
  90         u32 PGETBL_save;
  91         u32 __iomem *gtt;               /* I915G */
  92         int num_dcache_entries;
  93         union {
  94                 void __iomem *i9xx_flush_page;
  95                 void *i8xx_flush_page;
  96         };
  97         struct page *i8xx_page;
  98         struct resource ifp_resource;
  99         int resource_valid;
 100         struct page *scratch_page;
 101         dma_addr_t scratch_page_dma;
 102 } intel_private;
 103
 104 static int intel_fake_agp_insert_entries(struct agp_memory *mem,
 105                                          off_t pg_start, int type);
 106
 107 #define INTEL_GTT_GEN   intel_private.driver->gen
 108 #define IS_G33          intel_private.driver->is_g33
 109 #define IS_PINEVIEW     intel_private.driver->is_pineview
 110 #define IS_IRONLAKE     intel_private.driver->is_ironlake
 111 #define HAS_PGTBL_EN    intel_private.driver->has_pgtbl_enable
 112
 113 static void intel_agp_free_sglist(struct agp_memory *mem)
 114 {
 115         struct sg_table st;
 116
 117         st.sgl = mem->sg_list;
 118         st.orig_nents = st.nents = mem->page_count;
 119
 120         sg_free_table(&st);
 121
 122         mem->sg_list = NULL;
 123         mem->num_sg = 0;
 124 }
 125
 126 static int intel_agp_map_memory(struct agp_memory *mem)
 127 {
 128         struct sg_table st;
 129         struct scatterlist *sg;
 130         int i;
 131
 132         if (mem->sg_list)
 133                 return 0; /* already mapped (for e.g. resume */
 134
 135         DBG("try mapping %lu pages\n", (unsigned long)mem->page_count);
 136
 137         if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL))
 138                 goto err;
 139
 140         mem->sg_list = sg = st.sgl;
 141
 142         for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg))
 143                 sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0);
 144
 145         mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list,
 146                                  mem->page_count, PCI_DMA_BIDIRECTIONAL);
 147         if (unlikely(!mem->num_sg))
 148                 goto err;
 149
 150         return 0;
 151
 152 err:
 153         sg_free_table(&st);
 154         return -ENOMEM;
 155 }
 156
 157 static void intel_agp_unmap_memory(struct agp_memory *mem)
 158 {
 159         DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
 160
 161         pci_unmap_sg(intel_private.pcidev, mem->sg_list,
 162                      mem->page_count, PCI_DMA_BIDIRECTIONAL);
 163         intel_agp_free_sglist(mem);
 164 }
 165
 166 static int intel_i810_fetch_size(void)
 167 {
 168         u32 smram_miscc;
 169         struct aper_size_info_fixed *values;
 170
 171         pci_read_config_dword(intel_private.bridge_dev,
 172                               I810_SMRAM_MISCC, &smram_miscc);
 173         values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
 174
 175         if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) {
 176                 dev_warn(&intel_private.bridge_dev->dev, "i810 is disabled\n");
 177                 return 0;
 178         }
 179         if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) {
 180                 agp_bridge->current_size = (void *) (values + 1);
 181                 agp_bridge->aperture_size_idx = 1;
 182                 intel_private.base.gtt_total_entries = KB(32) / 4;
 183                 return values[1].size;
 184         } else {
 185                 agp_bridge->current_size = (void *) (values);
 186                 agp_bridge->aperture_size_idx = 0;
 187                 intel_private.base.gtt_total_entries = KB(64) / 4;
 188                 return values[0].size;
 189         }
 190
 191         return 0;
 192 }
 193
 194 static int intel_i810_configure(void)
 195 {
 196         struct aper_size_info_fixed *current_size;
 197         u32 temp;
 198         int i;
 199
 200         current_size = A_SIZE_FIX(agp_bridge->current_size);
 201
 202         if (!intel_private.registers) {
 203                 pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp);
 204                 temp &= 0xfff80000;
 205
 206                 intel_private.registers = ioremap(temp, 128 * 4096);
 207                 if (!intel_private.registers) {
 208                         dev_err(&intel_private.pcidev->dev,
 209                                 "can't remap memory\n");
 210                         return -ENOMEM;
 211                 }
 212         }
 213
 214         intel_private.gtt = intel_private.registers + I810_PTE_BASE;
 215         intel_private.scratch_page_dma = agp_bridge->scratch_page & PAGE_MASK;
 216
 217         if ((readl(intel_private.registers+I810_DRAM_CTL)
 218                 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
 219                 /* This will need to be dynamically assigned */
 220                 dev_info(&intel_private.pcidev->dev,
 221                          "detected 4MB dedicated video ram\n");
 222                 intel_private.num_dcache_entries = 1024;
 223         }
 224         pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
 225         agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
 226         writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
 227         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
 228
 229         if (agp_bridge->driver->needs_scratch_page) {
 230                 for (i = 0; i < current_size->num_entries; i++) {
 231                         writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 232                 }
 233                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); /* PCI posting. */
 234         }
 235         global_cache_flush();
 236         return 0;
 237 }
 238
 239 static void intel_i810_cleanup(void)
 240 {
 241         writel(0, intel_private.registers+I810_PGETBL_CTL);
 242         readl(intel_private.registers); /* PCI Posting. */
 243         iounmap(intel_private.registers);
 244 }
 245
 246 static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
 247 {
 248         return;
 249 }
 250
 251 /* Exists to support ARGB cursors */
 252 static struct page *i8xx_alloc_pages(void)
 253 {
 254         struct page *page;
 255
 256         page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2);
 257         if (page == NULL)
 258                 return NULL;
 259
 260         if (set_pages_uc(page, 4) < 0) {
 261                 set_pages_wb(page, 4);
 262                 __free_pages(page, 2);
 263                 return NULL;
 264         }
 265         get_page(page);
 266         atomic_inc(&agp_bridge->current_memory_agp);
 267         return page;
 268 }
 269
 270 static void i8xx_destroy_pages(struct page *page)
 271 {
 272         if (page == NULL)
 273                 return;
 274
 275         set_pages_wb(page, 4);
 276         put_page(page);
 277         __free_pages(page, 2);
 278         atomic_dec(&agp_bridge->current_memory_agp);
 279 }
 280
 281 static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start,
 282                                 int type)
 283 {
 284         int i;
 285
 286         if (type == AGP_DCACHE_MEMORY) {
 287                 if ((pg_start + mem->page_count)
 288                                 > intel_private.num_dcache_entries)
 289                         return -EINVAL;
 290
 291                 if (!mem->is_flushed)
 292                         global_cache_flush();
 293
 294                 for (i = pg_start; i < (pg_start + mem->page_count); i++) {
 295                         dma_addr_t addr = i << PAGE_SHIFT;
 296                         intel_private.driver->write_entry(addr,
 297                                                           i, type);
 298                 }
 299                 readl(intel_private.gtt+i-1);
 300
 301                 return 0;
 302         }
 303
 304         return intel_fake_agp_insert_entries(mem, pg_start, type);
 305 }
 306
 307 /*
 308  * The i810/i830 requires a physical address to program its mouse
 309  * pointer into hardware.
 310  * However the Xserver still writes to it through the agp aperture.
 311  */
 312 static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
 313 {
 314         struct agp_memory *new;
 315         struct page *page;
 316
 317         switch (pg_count) {
 318         case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge);
 319                 break;
 320         case 4:
 321                 /* kludge to get 4 physical pages for ARGB cursor */
 322                 page = i8xx_alloc_pages();
 323                 break;
 324         default:
 325                 return NULL;
 326         }
 327
 328         if (page == NULL)
 329                 return NULL;
 330
 331         new = agp_create_memory(pg_count);
 332         if (new == NULL)
 333                 return NULL;
 334
 335         new->pages[0] = page;
 336         if (pg_count == 4) {
 337                 /* kludge to get 4 physical pages for ARGB cursor */
 338                 new->pages[1] = new->pages[0] + 1;
 339                 new->pages[2] = new->pages[1] + 1;
 340                 new->pages[3] = new->pages[2] + 1;
 341         }
 342         new->page_count = pg_count;
 343         new->num_scratch_pages = pg_count;
 344         new->type = AGP_PHYS_MEMORY;
 345         new->physical = page_to_phys(new->pages[0]);
 346         return new;
 347 }
 348
 349 static void intel_i810_free_by_type(struct agp_memory *curr)
 350 {
 351         agp_free_key(curr->key);
 352         if (curr->type == AGP_PHYS_MEMORY) {
 353                 if (curr->page_count == 4)
 354                         i8xx_destroy_pages(curr->pages[0]);
 355                 else {
 356                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 357                                                              AGP_PAGE_DESTROY_UNMAP);
 358                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 359                                                              AGP_PAGE_DESTROY_FREE);
 360                 }
 361                 agp_free_page_array(curr);
 362         }
 363         kfree(curr);
 364 }
 365
 366 static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
 367                                             dma_addr_t addr, int type)
 368 {
 369         /* Type checking must be done elsewhere */
 370         return addr | bridge->driver->masks[type].mask;
 371 }
 372
 373 static int intel_gtt_setup_scratch_page(void)
 374 {
 375         struct page *page;
 376         dma_addr_t dma_addr;
 377
 378         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
 379         if (page == NULL)
 380                 return -ENOMEM;
 381         get_page(page);
 382         set_pages_uc(page, 1);
 383
 384         if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
 385                 dma_addr = pci_map_page(intel_private.pcidev, page, 0,
 386                                     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 387                 if (pci_dma_mapping_error(intel_private.pcidev, dma_addr))
 388                         return -EINVAL;
 389
 390                 intel_private.scratch_page_dma = dma_addr;
 391         } else
 392                 intel_private.scratch_page_dma = page_to_phys(page);
 393
 394         intel_private.scratch_page = page;
 395
 396         return 0;
 397 }
 398
 399 static void i810_write_entry(dma_addr_t addr, unsigned int entry,
 400                              unsigned int flags)
 401 {
 402         u32 pte_flags = I810_PTE_VALID;
 403
 404         switch (flags) {
 405         case AGP_DCACHE_MEMORY:
 406                 pte_flags |= I810_PTE_LOCAL;
 407                 break;
 408         case AGP_USER_CACHED_MEMORY:
 409                 pte_flags |= I830_PTE_SYSTEM_CACHED;
 410                 break;
 411         }
 412
 413         writel(addr | pte_flags, intel_private.gtt + entry);
 414 }
 415
 416 static const struct aper_size_info_fixed const intel_fake_agp_sizes[] = {
 417         {128, 32768, 5},
 418         /* The 64M mode still requires a 128k gatt */
 419         {64, 16384, 5},
 420         {256, 65536, 6},
 421         {512, 131072, 7},
 422 };
 423
 424 static unsigned int intel_gtt_stolen_size(void)
 425 {
 426         u16 gmch_ctrl;
 427         u8 rdct;
 428         int local = 0;
 429         static const int ddt[4] = { 0, 16, 32, 64 };
 430         unsigned int stolen_size = 0;
 431
 432         pci_read_config_word(intel_private.bridge_dev,
 433                              I830_GMCH_CTRL, &gmch_ctrl);
 434
 435         if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
 436             intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
 437                 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
 438                 case I830_GMCH_GMS_STOLEN_512:
 439                         stolen_size = KB(512);
 440                         break;
 441                 case I830_GMCH_GMS_STOLEN_1024:
 442                         stolen_size = MB(1);
 443                         break;
 444                 case I830_GMCH_GMS_STOLEN_8192:
 445                         stolen_size = MB(8);
 446                         break;
 447                 case I830_GMCH_GMS_LOCAL:
 448                         rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE);
 449                         stolen_size = (I830_RDRAM_ND(rdct) + 1) *
 450                                         MB(ddt[I830_RDRAM_DDT(rdct)]);
 451                         local = 1;
 452                         break;
 453                 default:
 454                         stolen_size = 0;
 455                         break;
 456                 }
 457         } else if (INTEL_GTT_GEN == 6) {
 458                 /*
 459                  * SandyBridge has new memory control reg at 0x50.w
 460                  */
 461                 u16 snb_gmch_ctl;
 462                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 463                 switch (snb_gmch_ctl & SNB_GMCH_GMS_STOLEN_MASK) {
 464                 case SNB_GMCH_GMS_STOLEN_32M:
 465                         stolen_size = MB(32);
 466                         break;
 467                 case SNB_GMCH_GMS_STOLEN_64M:
 468                         stolen_size = MB(64);
 469                         break;
 470                 case SNB_GMCH_GMS_STOLEN_96M:
 471                         stolen_size = MB(96);
 472                         break;
 473                 case SNB_GMCH_GMS_STOLEN_128M:
 474                         stolen_size = MB(128);
 475                         break;
 476                 case SNB_GMCH_GMS_STOLEN_160M:
 477                         stolen_size = MB(160);
 478                         break;
 479                 case SNB_GMCH_GMS_STOLEN_192M:
 480                         stolen_size = MB(192);
 481                         break;
 482                 case SNB_GMCH_GMS_STOLEN_224M:
 483                         stolen_size = MB(224);
 484                         break;
 485                 case SNB_GMCH_GMS_STOLEN_256M:
 486                         stolen_size = MB(256);
 487                         break;
 488                 case SNB_GMCH_GMS_STOLEN_288M:
 489                         stolen_size = MB(288);
 490                         break;
 491                 case SNB_GMCH_GMS_STOLEN_320M:
 492                         stolen_size = MB(320);
 493                         break;
 494                 case SNB_GMCH_GMS_STOLEN_352M:
 495                         stolen_size = MB(352);
 496                         break;
 497                 case SNB_GMCH_GMS_STOLEN_384M:
 498                         stolen_size = MB(384);
 499                         break;
 500                 case SNB_GMCH_GMS_STOLEN_416M:
 501                         stolen_size = MB(416);
 502                         break;
 503                 case SNB_GMCH_GMS_STOLEN_448M:
 504                         stolen_size = MB(448);
 505                         break;
 506                 case SNB_GMCH_GMS_STOLEN_480M:
 507                         stolen_size = MB(480);
 508                         break;
 509                 case SNB_GMCH_GMS_STOLEN_512M:
 510                         stolen_size = MB(512);
 511                         break;
 512                 }
 513         } else {
 514                 switch (gmch_ctrl & I855_GMCH_GMS_MASK) {
 515                 case I855_GMCH_GMS_STOLEN_1M:
 516                         stolen_size = MB(1);
 517                         break;
 518                 case I855_GMCH_GMS_STOLEN_4M:
 519                         stolen_size = MB(4);
 520                         break;
 521                 case I855_GMCH_GMS_STOLEN_8M:
 522                         stolen_size = MB(8);
 523                         break;
 524                 case I855_GMCH_GMS_STOLEN_16M:
 525                         stolen_size = MB(16);
 526                         break;
 527                 case I855_GMCH_GMS_STOLEN_32M:
 528                         stolen_size = MB(32);
 529                         break;
 530                 case I915_GMCH_GMS_STOLEN_48M:
 531                         stolen_size = MB(48);
 532                         break;
 533                 case I915_GMCH_GMS_STOLEN_64M:
 534                         stolen_size = MB(64);
 535                         break;
 536                 case G33_GMCH_GMS_STOLEN_128M:
 537                         stolen_size = MB(128);
 538                         break;
 539                 case G33_GMCH_GMS_STOLEN_256M:
 540                         stolen_size = MB(256);
 541                         break;
 542                 case INTEL_GMCH_GMS_STOLEN_96M:
 543                         stolen_size = MB(96);
 544                         break;
 545                 case INTEL_GMCH_GMS_STOLEN_160M:
 546                         stolen_size = MB(160);
 547                         break;
 548                 case INTEL_GMCH_GMS_STOLEN_224M:
 549                         stolen_size = MB(224);
 550                         break;
 551                 case INTEL_GMCH_GMS_STOLEN_352M:
 552                         stolen_size = MB(352);
 553                         break;
 554                 default:
 555                         stolen_size = 0;
 556                         break;
 557                 }
 558         }
 559
 560         if (stolen_size > 0) {
 561                 dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
 562                        stolen_size / KB(1), local ? "local" : "stolen");
 563         } else {
 564                 dev_info(&intel_private.bridge_dev->dev,
 565                        "no pre-allocated video memory detected\n");
 566                 stolen_size = 0;
 567         }
 568
 569         return stolen_size;
 570 }
 571
 572 static void i965_adjust_pgetbl_size(unsigned int size_flag)
 573 {
 574         u32 pgetbl_ctl, pgetbl_ctl2;
 575
 576         /* ensure that ppgtt is disabled */
 577         pgetbl_ctl2 = readl(intel_private.registers+I965_PGETBL_CTL2);
 578         pgetbl_ctl2 &= ~I810_PGETBL_ENABLED;
 579         writel(pgetbl_ctl2, intel_private.registers+I965_PGETBL_CTL2);
 580
 581         /* write the new ggtt size */
 582         pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
 583         pgetbl_ctl &= ~I965_PGETBL_SIZE_MASK;
 584         pgetbl_ctl |= size_flag;
 585         writel(pgetbl_ctl, intel_private.registers+I810_PGETBL_CTL);
 586 }
 587
 588 static unsigned int i965_gtt_total_entries(void)
 589 {
 590         int size;
 591         u32 pgetbl_ctl;
 592         u16 gmch_ctl;
 593
 594         pci_read_config_word(intel_private.bridge_dev,
 595                              I830_GMCH_CTRL, &gmch_ctl);
 596
 597         if (INTEL_GTT_GEN == 5) {
 598                 switch (gmch_ctl & G4x_GMCH_SIZE_MASK) {
 599                 case G4x_GMCH_SIZE_1M:
 600                 case G4x_GMCH_SIZE_VT_1M:
 601                         i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1MB);
 602                         break;
 603                 case G4x_GMCH_SIZE_VT_1_5M:
 604                         i965_adjust_pgetbl_size(I965_PGETBL_SIZE_1_5MB);
 605                         break;
 606                 case G4x_GMCH_SIZE_2M:
 607                 case G4x_GMCH_SIZE_VT_2M:
 608                         i965_adjust_pgetbl_size(I965_PGETBL_SIZE_2MB);
 609                         break;
 610                 }
 611         }
 612
 613         pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
 614
 615         switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
 616         case I965_PGETBL_SIZE_128KB:
 617                 size = KB(128);
 618                 break;
 619         case I965_PGETBL_SIZE_256KB:
 620                 size = KB(256);
 621                 break;
 622         case I965_PGETBL_SIZE_512KB:
 623                 size = KB(512);
 624                 break;
 625         /* GTT pagetable sizes bigger than 512KB are not possible on G33! */
 626         case I965_PGETBL_SIZE_1MB:
 627                 size = KB(1024);
 628                 break;
 629         case I965_PGETBL_SIZE_2MB:
 630                 size = KB(2048);
 631                 break;
 632         case I965_PGETBL_SIZE_1_5MB:
 633                 size = KB(1024 + 512);
 634                 break;
 635         default:
 636                 dev_info(&intel_private.pcidev->dev,
 637                          "unknown page table size, assuming 512KB\n");
 638                 size = KB(512);
 639         }
 640
 641         return size/4;
 642 }
 643
 644 static unsigned int intel_gtt_total_entries(void)
 645 {
 646         int size;
 647
 648         if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5)
 649                 return i965_gtt_total_entries();
 650         else if (INTEL_GTT_GEN == 6) {
 651                 u16 snb_gmch_ctl;
 652
 653                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 654                 switch (snb_gmch_ctl & SNB_GTT_SIZE_MASK) {
 655                 default:
 656                 case SNB_GTT_SIZE_0M:
 657                         printk(KERN_ERR "Bad GTT size mask: 0x%04x.\n", snb_gmch_ctl);
 658                         size = MB(0);
 659                         break;
 660                 case SNB_GTT_SIZE_1M:
 661                         size = MB(1);
 662                         break;
 663                 case SNB_GTT_SIZE_2M:
 664                         size = MB(2);
 665                         break;
 666                 }
 667                 return size/4;
 668         } else {
 669                 /* On previous hardware, the GTT size was just what was
 670                  * required to map the aperture.
 671                  */
 672                 return intel_private.base.gtt_mappable_entries;
 673         }
 674 }
 675
 676 static unsigned int intel_gtt_mappable_entries(void)
 677 {
 678         unsigned int aperture_size;
 679
 680         if (INTEL_GTT_GEN == 2) {
 681                 u16 gmch_ctrl;
 682
 683                 pci_read_config_word(intel_private.bridge_dev,
 684                                      I830_GMCH_CTRL, &gmch_ctrl);
 685
 686                 if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_64M)
 687                         aperture_size = MB(64);
 688                 else
 689                         aperture_size = MB(128);
 690         } else {
 691                 /* 9xx supports large sizes, just look at the length */
 692                 aperture_size = pci_resource_len(intel_private.pcidev, 2);
 693         }
 694
 695         return aperture_size >> PAGE_SHIFT;
 696 }
 697
 698 static void intel_gtt_teardown_scratch_page(void)
 699 {
 700         set_pages_wb(intel_private.scratch_page, 1);
 701         pci_unmap_page(intel_private.pcidev, intel_private.scratch_page_dma,
 702                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 703         put_page(intel_private.scratch_page);
 704         __free_page(intel_private.scratch_page);
 705 }
 706
 707 static void intel_gtt_cleanup(void)
 708 {
 709         intel_private.driver->cleanup();
 710
 711         iounmap(intel_private.gtt);
 712         iounmap(intel_private.registers);
 713
 714         intel_gtt_teardown_scratch_page();
 715 }
 716
 717 static int intel_gtt_init(void)
 718 {
 719         u32 gtt_map_size;
 720         int ret;
 721
 722         ret = intel_private.driver->setup();
 723         if (ret != 0)
 724                 return ret;
 725
 726         intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries();
 727         intel_private.base.gtt_total_entries = intel_gtt_total_entries();
 728
 729         /* save the PGETBL reg for resume */
 730         intel_private.PGETBL_save =
 731                 readl(intel_private.registers+I810_PGETBL_CTL)
 732                         & ~I810_PGETBL_ENABLED;
 733         /* we only ever restore the register when enabling the PGTBL... */
 734         if (HAS_PGTBL_EN)
 735                 intel_private.PGETBL_save |= I810_PGETBL_ENABLED;
 736
 737         dev_info(&intel_private.bridge_dev->dev,
 738                         "detected gtt size: %dK total, %dK mappable\n",
 739                         intel_private.base.gtt_total_entries * 4,
 740                         intel_private.base.gtt_mappable_entries * 4);
 741
 742         gtt_map_size = intel_private.base.gtt_total_entries * 4;
 743
 744         intel_private.gtt = ioremap(intel_private.gtt_bus_addr,
 745                                     gtt_map_size);
 746         if (!intel_private.gtt) {
 747                 intel_private.driver->cleanup();
 748                 iounmap(intel_private.registers);
 749                 return -ENOMEM;
 750         }
 751
 752         global_cache_flush();   /* FIXME: ? */
 753
 754         /* we have to call this as early as possible after the MMIO base address is known */
 755         intel_private.base.stolen_size = intel_gtt_stolen_size();
 756         if (intel_private.base.stolen_size == 0) {
 757                 intel_private.driver->cleanup();
 758                 iounmap(intel_private.registers);
 759                 iounmap(intel_private.gtt);
 760                 return -ENOMEM;
 761         }
 762
 763         ret = intel_gtt_setup_scratch_page();
 764         if (ret != 0) {
 765                 intel_gtt_cleanup();
 766                 return ret;
 767         }
 768
 769         return 0;
 770 }
 771
 772 static int intel_fake_agp_fetch_size(void)
 773 {
 774         int num_sizes = ARRAY_SIZE(intel_fake_agp_sizes);
 775         unsigned int aper_size;
 776         int i;
 777
 778         aper_size = (intel_private.base.gtt_mappable_entries << PAGE_SHIFT)
 779                     / MB(1);
 780
 781         for (i = 0; i < num_sizes; i++) {
 782                 if (aper_size == intel_fake_agp_sizes[i].size) {
 783                         agp_bridge->current_size =
 784                                 (void *) (intel_fake_agp_sizes + i);
 785                         return aper_size;
 786                 }
 787         }
 788
 789         return 0;
 790 }
 791
 792 static void i830_cleanup(void)
 793 {
 794         kunmap(intel_private.i8xx_page);
 795         intel_private.i8xx_flush_page = NULL;
 796
 797         __free_page(intel_private.i8xx_page);
 798         intel_private.i8xx_page = NULL;
 799 }
 800
 801 static void intel_i830_setup_flush(void)
 802 {
 803         /* return if we've already set the flush mechanism up */
 804         if (intel_private.i8xx_page)
 805                 return;
 806
 807         intel_private.i8xx_page = alloc_page(GFP_KERNEL);
 808         if (!intel_private.i8xx_page)
 809                 return;
 810
 811         intel_private.i8xx_flush_page = kmap(intel_private.i8xx_page);
 812         if (!intel_private.i8xx_flush_page)
 813                 i830_cleanup();
 814 }
 815
 816 /* The chipset_flush interface needs to get data that has already been
 817  * flushed out of the CPU all the way out to main memory, because the GPU
 818  * doesn't snoop those buffers.
 819  *
 820  * The 8xx series doesn't have the same lovely interface for flushing the
 821  * chipset write buffers that the later chips do. According to the 865
 822  * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
 823  * that buffer out, we just fill 1KB and clflush it out, on the assumption
 824  * that it'll push whatever was in there out.  It appears to work.
 825  */
 826 static void i830_chipset_flush(void)
 827 {
 828         unsigned int *pg = intel_private.i8xx_flush_page;
 829
 830         memset(pg, 0, 1024);
 831
 832         if (cpu_has_clflush)
 833                 clflush_cache_range(pg, 1024);
 834         else if (wbinvd_on_all_cpus() != 0)
 835                 printk(KERN_ERR "Timed out waiting for cache flush.\n");
 836 }
 837
 838 static void i830_write_entry(dma_addr_t addr, unsigned int entry,
 839                              unsigned int flags)
 840 {
 841         u32 pte_flags = I810_PTE_VALID;
 842
 843         if (flags ==  AGP_USER_CACHED_MEMORY)
 844                 pte_flags |= I830_PTE_SYSTEM_CACHED;
 845
 846         writel(addr | pte_flags, intel_private.gtt + entry);
 847 }
 848
 849 static bool intel_enable_gtt(void)
 850 {
 851         u32 gma_addr;
 852         u8 __iomem *reg;
 853
 854         if (INTEL_GTT_GEN == 2)
 855                 pci_read_config_dword(intel_private.pcidev, I810_GMADDR,
 856                                       &gma_addr);
 857         else
 858                 pci_read_config_dword(intel_private.pcidev, I915_GMADDR,
 859                                       &gma_addr);
 860
 861         intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK);
 862
 863         if (INTEL_GTT_GEN >= 6)
 864             return true;
 865
 866         if (INTEL_GTT_GEN == 2) {
 867                 u16 gmch_ctrl;
 868
 869                 pci_read_config_word(intel_private.bridge_dev,
 870                                      I830_GMCH_CTRL, &gmch_ctrl);
 871                 gmch_ctrl |= I830_GMCH_ENABLED;
 872                 pci_write_config_word(intel_private.bridge_dev,
 873                                       I830_GMCH_CTRL, gmch_ctrl);
 874
 875                 pci_read_config_word(intel_private.bridge_dev,
 876                                      I830_GMCH_CTRL, &gmch_ctrl);
 877                 if ((gmch_ctrl & I830_GMCH_ENABLED) == 0) {
 878                         dev_err(&intel_private.pcidev->dev,
 879                                 "failed to enable the GTT: GMCH_CTRL=%x\n",
 880                                 gmch_ctrl);
 881                         return false;
 882                 }
 883         }
 884
 885         reg = intel_private.registers+I810_PGETBL_CTL;
 886         writel(intel_private.PGETBL_save, reg);
 887         if (HAS_PGTBL_EN && (readl(reg) & I810_PGETBL_ENABLED) == 0) {
 888                 dev_err(&intel_private.pcidev->dev,
 889                         "failed to enable the GTT: PGETBL=%x [expected %x]\n",
 890                         readl(reg), intel_private.PGETBL_save);
 891                 return false;
 892         }
 893
 894         return true;
 895 }
 896
 897 static int i830_setup(void)
 898 {
 899         u32 reg_addr;
 900
 901         pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
 902         reg_addr &= 0xfff80000;
 903
 904         intel_private.registers = ioremap(reg_addr, KB(64));
 905         if (!intel_private.registers)
 906                 return -ENOMEM;
 907
 908         intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
 909
 910         intel_i830_setup_flush();
 911
 912         return 0;
 913 }
 914
 915 static int intel_fake_agp_create_gatt_table(struct agp_bridge_data *bridge)
 916 {
 917         agp_bridge->gatt_table_real = NULL;
 918         agp_bridge->gatt_table = NULL;
 919         agp_bridge->gatt_bus_addr = 0;
 920
 921         return 0;
 922 }
 923
 924 static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge)
 925 {
 926         return 0;
 927 }
 928
 929 static int intel_fake_agp_configure(void)
 930 {
 931         int i;
 932
 933         if (!intel_enable_gtt())
 934             return -EIO;
 935
 936         agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;
 937
 938         for (i = 0; i < intel_private.base.gtt_total_entries; i++) {
 939                 intel_private.driver->write_entry(intel_private.scratch_page_dma,
 940                                                   i, 0);
 941         }
 942         readl(intel_private.gtt+i-1);   /* PCI Posting. */
 943
 944         global_cache_flush();
 945
 946         return 0;
 947 }
 948
 949 static bool i830_check_flags(unsigned int flags)
 950 {
 951         switch (flags) {
 952         case 0:
 953         case AGP_PHYS_MEMORY:
 954         case AGP_USER_CACHED_MEMORY:
 955         case AGP_USER_MEMORY:
 956                 return true;
 957         }
 958
 959         return false;
 960 }
 961
 962 static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
 963                                         unsigned int sg_len,
 964                                         unsigned int pg_start,
 965                                         unsigned int flags)
 966 {
 967         struct scatterlist *sg;
 968         unsigned int len, m;
 969         int i, j;
 970
 971         j = pg_start;
 972
 973         /* sg may merge pages, but we have to separate
 974          * per-page addr for GTT */
 975         for_each_sg(sg_list, sg, sg_len, i) {
 976                 len = sg_dma_len(sg) >> PAGE_SHIFT;
 977                 for (m = 0; m < len; m++) {
 978                         dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 979                         intel_private.driver->write_entry(addr,
 980                                                           j, flags);
 981                         j++;
 982                 }
 983         }
 984         readl(intel_private.gtt+j-1);
 985 }
 986
 987 static int intel_fake_agp_insert_entries(struct agp_memory *mem,
 988                                          off_t pg_start, int type)
 989 {
 990         int i, j;
 991         int ret = -EINVAL;
 992
 993         if (mem->page_count == 0)
 994                 goto out;
 995
 996         if (pg_start + mem->page_count > intel_private.base.gtt_total_entries)
 997                 goto out_err;
 998
 999         if (type != mem->type)
1000                 goto out_err;
1001
1002         if (!intel_private.driver->check_flags(type))
1003                 goto out_err;
1004
1005         if (!mem->is_flushed)
1006                 global_cache_flush();
1007
1008         if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
1009                 ret = intel_agp_map_memory(mem);
1010                 if (ret != 0)
1011                         return ret;
1012
1013                 intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg,
1014                                             pg_start, type);
1015         } else {
1016                 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
1017                         dma_addr_t addr = page_to_phys(mem->pages[i]);
1018                         intel_private.driver->write_entry(addr,
1019                                                           j, type);
1020                 }
1021                 readl(intel_private.gtt+j-1);
1022         }
1023
1024 out:
1025         ret = 0;
1026 out_err:
1027         mem->is_flushed = true;
1028         return ret;
1029 }
1030
1031 static int intel_fake_agp_remove_entries(struct agp_memory *mem,
1032                                          off_t pg_start, int type)
1033 {
1034         int i;
1035
1036         if (mem->page_count == 0)
1037                 return 0;
1038
1039         if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2)
1040                 intel_agp_unmap_memory(mem);
1041
1042         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
1043                 intel_private.driver->write_entry(intel_private.scratch_page_dma,
1044                                                   i, 0);
1045         }
1046         readl(intel_private.gtt+i-1);
1047
1048         return 0;
1049 }
1050
1051 static void intel_fake_agp_chipset_flush(struct agp_bridge_data *bridge)
1052 {
1053         intel_private.driver->chipset_flush();
1054 }
1055
1056 static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count,
1057                                                        int type)
1058 {
1059         struct agp_memory *new;
1060
1061         if (type == AGP_DCACHE_MEMORY && INTEL_GTT_GEN == 1) {
1062                 if (pg_count != intel_private.num_dcache_entries)
1063                         return NULL;
1064
1065                 new = agp_create_memory(1);
1066                 if (new == NULL)
1067                         return NULL;
1068
1069                 new->type = AGP_DCACHE_MEMORY;
1070                 new->page_count = pg_count;
1071                 new->num_scratch_pages = 0;
1072                 agp_free_page_array(new);
1073                 return new;
1074         }
1075         if (type == AGP_PHYS_MEMORY)
1076                 return alloc_agpphysmem_i8xx(pg_count, type);
1077         /* always return NULL for other allocation types for now */
1078         return NULL;
1079 }
1080
1081 static int intel_alloc_chipset_flush_resource(void)
1082 {
1083         int ret;
1084         ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE,
1085                                      PAGE_SIZE, PCIBIOS_MIN_MEM, 0,
1086                                      pcibios_align_resource, intel_private.bridge_dev);
1087
1088         return ret;
1089 }
1090
1091 static void intel_i915_setup_chipset_flush(void)
1092 {
1093         int ret;
1094         u32 temp;
1095
1096         pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp);
1097         if (!(temp & 0x1)) {
1098                 intel_alloc_chipset_flush_resource();
1099                 intel_private.resource_valid = 1;
1100                 pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1101         } else {
1102                 temp &= ~1;
1103
1104                 intel_private.resource_valid = 1;
1105                 intel_private.ifp_resource.start = temp;
1106                 intel_private.ifp_resource.end = temp + PAGE_SIZE;
1107                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1108                 /* some BIOSes reserve this area in a pnp some don't */
1109                 if (ret)
1110                         intel_private.resource_valid = 0;
1111         }
1112 }
1113
1114 static void intel_i965_g33_setup_chipset_flush(void)
1115 {
1116         u32 temp_hi, temp_lo;
1117         int ret;
1118
1119         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi);
1120         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo);
1121
1122         if (!(temp_lo & 0x1)) {
1123
1124                 intel_alloc_chipset_flush_resource();
1125
1126                 intel_private.resource_valid = 1;
1127                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4,
1128                         upper_32_bits(intel_private.ifp_resource.start));
1129                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1130         } else {
1131                 u64 l64;
1132
1133                 temp_lo &= ~0x1;
1134                 l64 = ((u64)temp_hi << 32) | temp_lo;
1135
1136                 intel_private.resource_valid = 1;
1137                 intel_private.ifp_resource.start = l64;
1138                 intel_private.ifp_resource.end = l64 + PAGE_SIZE;
1139                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1140                 /* some BIOSes reserve this area in a pnp some don't */
1141                 if (ret)
1142                         intel_private.resource_valid = 0;
1143         }
1144 }
1145
1146 static void intel_i9xx_setup_flush(void)
1147 {
1148         /* return if already configured */
1149         if (intel_private.ifp_resource.start)
1150                 return;
1151
1152         if (INTEL_GTT_GEN == 6)
1153                 return;
1154
1155         /* setup a resource for this object */
1156         intel_private.ifp_resource.name = "Intel Flush Page";
1157         intel_private.ifp_resource.flags = IORESOURCE_MEM;
1158
1159         /* Setup chipset flush for 915 */
1160         if (IS_G33 || INTEL_GTT_GEN >= 4) {
1161                 intel_i965_g33_setup_chipset_flush();
1162         } else {
1163                 intel_i915_setup_chipset_flush();
1164         }
1165
1166         if (intel_private.ifp_resource.start)
1167                 intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE);
1168         if (!intel_private.i9xx_flush_page)
1169                 dev_err(&intel_private.pcidev->dev,
1170                         "can't ioremap flush page - no chipset flushing\n");
1171 }
1172
1173 static void i9xx_cleanup(void)
1174 {
1175         if (intel_private.i9xx_flush_page)
1176                 iounmap(intel_private.i9xx_flush_page);
1177         if (intel_private.resource_valid)
1178                 release_resource(&intel_private.ifp_resource);
1179         intel_private.ifp_resource.start = 0;
1180         intel_private.resource_valid = 0;
1181 }
1182
1183 static void i9xx_chipset_flush(void)
1184 {
1185         if (intel_private.i9xx_flush_page)
1186                 writel(1, intel_private.i9xx_flush_page);
1187 }
1188
1189 static void i965_write_entry(dma_addr_t addr, unsigned int entry,
1190                              unsigned int flags)
1191 {
1192         /* Shift high bits down */
1193         addr |= (addr >> 28) & 0xf0;
1194         writel(addr | I810_PTE_VALID, intel_private.gtt + entry);
1195 }
1196
1197 static bool gen6_check_flags(unsigned int flags)
1198 {
1199         return true;
1200 }
1201
1202 static void gen6_write_entry(dma_addr_t addr, unsigned int entry,
1203                              unsigned int flags)
1204 {
1205         unsigned int type_mask = flags & ~AGP_USER_CACHED_MEMORY_GFDT;
1206         unsigned int gfdt = flags & AGP_USER_CACHED_MEMORY_GFDT;
1207         u32 pte_flags;
1208
1209         if (type_mask == AGP_USER_MEMORY)
1210                 pte_flags = GEN6_PTE_UNCACHED | I810_PTE_VALID;
1211         else if (type_mask == AGP_USER_CACHED_MEMORY_LLC_MLC) {
1212                 pte_flags = GEN6_PTE_LLC_MLC | I810_PTE_VALID;
1213                 if (gfdt)
1214                         pte_flags |= GEN6_PTE_GFDT;
1215         } else { /* set 'normal'/'cached' to LLC by default */
1216                 pte_flags = GEN6_PTE_LLC | I810_PTE_VALID;
1217                 if (gfdt)
1218                         pte_flags |= GEN6_PTE_GFDT;
1219         }
1220
1221         /* gen6 has bit11-4 for physical addr bit39-32 */
1222         addr |= (addr >> 28) & 0xff0;
1223         writel(addr | pte_flags, intel_private.gtt + entry);
1224 }
1225
1226 static void gen6_cleanup(void)
1227 {
1228 }
1229
1230 static int i9xx_setup(void)
1231 {
1232         u32 reg_addr;
1233
1234         pci_read_config_dword(intel_private.pcidev, I915_MMADDR, &reg_addr);
1235
1236         reg_addr &= 0xfff80000;
1237
1238         intel_private.registers = ioremap(reg_addr, 128 * 4096);
1239         if (!intel_private.registers)
1240                 return -ENOMEM;
1241
1242         if (INTEL_GTT_GEN == 3) {
1243                 u32 gtt_addr;
1244
1245                 pci_read_config_dword(intel_private.pcidev,
1246                                       I915_PTEADDR, &gtt_addr);
1247                 intel_private.gtt_bus_addr = gtt_addr;
1248         } else {
1249                 u32 gtt_offset;
1250
1251                 switch (INTEL_GTT_GEN) {
1252                 case 5:
1253                 case 6:
1254                         gtt_offset = MB(2);
1255                         break;
1256                 case 4:
1257                 default:
1258                         gtt_offset =  KB(512);
1259                         break;
1260                 }
1261                 intel_private.gtt_bus_addr = reg_addr + gtt_offset;
1262         }
1263
1264         intel_i9xx_setup_flush();
1265
1266         return 0;
1267 }
1268
1269 static const struct agp_bridge_driver intel_810_driver = {
1270         .owner                  = THIS_MODULE,
1271         .aperture_sizes         = intel_i810_sizes,
1272         .size_type              = FIXED_APER_SIZE,
1273         .num_aperture_sizes     = 2,
1274         .needs_scratch_page     = true,
1275         .configure              = intel_i810_configure,
1276         .fetch_size             = intel_i810_fetch_size,
1277         .cleanup                = intel_i810_cleanup,
1278         .mask_memory            = intel_i810_mask_memory,
1279         .masks                  = intel_i810_masks,
1280         .agp_enable             = intel_fake_agp_enable,
1281         .cache_flush            = global_cache_flush,
1282         .create_gatt_table      = agp_generic_create_gatt_table,
1283         .free_gatt_table        = agp_generic_free_gatt_table,
1284         .insert_memory          = intel_i810_insert_entries,
1285         .remove_memory          = intel_fake_agp_remove_entries,
1286         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1287         .free_by_type           = intel_i810_free_by_type,
1288         .agp_alloc_page         = agp_generic_alloc_page,
1289         .agp_alloc_pages        = agp_generic_alloc_pages,
1290         .agp_destroy_page       = agp_generic_destroy_page,
1291         .agp_destroy_pages      = agp_generic_destroy_pages,
1292         .agp_type_to_mask_type  = agp_generic_type_to_mask_type,
1293 };
1294
1295 static const struct agp_bridge_driver intel_fake_agp_driver = {
1296         .owner                  = THIS_MODULE,
1297         .size_type              = FIXED_APER_SIZE,
1298         .aperture_sizes         = intel_fake_agp_sizes,
1299         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1300         .configure              = intel_fake_agp_configure,
1301         .fetch_size             = intel_fake_agp_fetch_size,
1302         .cleanup                = intel_gtt_cleanup,
1303         .agp_enable             = intel_fake_agp_enable,
1304         .cache_flush            = global_cache_flush,
1305         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1306         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1307         .insert_memory          = intel_fake_agp_insert_entries,
1308         .remove_memory          = intel_fake_agp_remove_entries,
1309         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1310         .free_by_type           = intel_i810_free_by_type,
1311         .agp_alloc_page         = agp_generic_alloc_page,
1312         .agp_alloc_pages        = agp_generic_alloc_pages,
1313         .agp_destroy_page       = agp_generic_destroy_page,
1314         .agp_destroy_pages      = agp_generic_destroy_pages,
1315         .chipset_flush          = intel_fake_agp_chipset_flush,
1316 };
1317
1318 static const struct intel_gtt_driver i81x_gtt_driver = {
1319         .gen = 1,
1320         .dma_mask_size = 32,
1321         .check_flags = i830_check_flags,
1322         .write_entry = i810_write_entry,
1323 };
1324 static const struct intel_gtt_driver i8xx_gtt_driver = {
1325         .gen = 2,
1326         .has_pgtbl_enable = 1,
1327         .setup = i830_setup,
1328         .cleanup = i830_cleanup,
1329         .write_entry = i830_write_entry,
1330         .dma_mask_size = 32,
1331         .check_flags = i830_check_flags,
1332         .chipset_flush = i830_chipset_flush,
1333 };
1334 static const struct intel_gtt_driver i915_gtt_driver = {
1335         .gen = 3,
1336         .has_pgtbl_enable = 1,
1337         .setup = i9xx_setup,
1338         .cleanup = i9xx_cleanup,
1339         /* i945 is the last gpu to need phys mem (for overlay and cursors). */
1340         .write_entry = i830_write_entry,
1341         .dma_mask_size = 32,
1342         .check_flags = i830_check_flags,
1343         .chipset_flush = i9xx_chipset_flush,
1344 };
1345 static const struct intel_gtt_driver g33_gtt_driver = {
1346         .gen = 3,
1347         .is_g33 = 1,
1348         .setup = i9xx_setup,
1349         .cleanup = i9xx_cleanup,
1350         .write_entry = i965_write_entry,
1351         .dma_mask_size = 36,
1352         .check_flags = i830_check_flags,
1353         .chipset_flush = i9xx_chipset_flush,
1354 };
1355 static const struct intel_gtt_driver pineview_gtt_driver = {
1356         .gen = 3,
1357         .is_pineview = 1, .is_g33 = 1,
1358         .setup = i9xx_setup,
1359         .cleanup = i9xx_cleanup,
1360         .write_entry = i965_write_entry,
1361         .dma_mask_size = 36,
1362         .check_flags = i830_check_flags,
1363         .chipset_flush = i9xx_chipset_flush,
1364 };
1365 static const struct intel_gtt_driver i965_gtt_driver = {
1366         .gen = 4,
1367         .has_pgtbl_enable = 1,
1368         .setup = i9xx_setup,
1369         .cleanup = i9xx_cleanup,
1370         .write_entry = i965_write_entry,
1371         .dma_mask_size = 36,
1372         .check_flags = i830_check_flags,
1373         .chipset_flush = i9xx_chipset_flush,
1374 };
1375 static const struct intel_gtt_driver g4x_gtt_driver = {
1376         .gen = 5,
1377         .setup = i9xx_setup,
1378         .cleanup = i9xx_cleanup,
1379         .write_entry = i965_write_entry,
1380         .dma_mask_size = 36,
1381         .check_flags = i830_check_flags,
1382         .chipset_flush = i9xx_chipset_flush,
1383 };
1384 static const struct intel_gtt_driver ironlake_gtt_driver = {
1385         .gen = 5,
1386         .is_ironlake = 1,
1387         .setup = i9xx_setup,
1388         .cleanup = i9xx_cleanup,
1389         .write_entry = i965_write_entry,
1390         .dma_mask_size = 36,
1391         .check_flags = i830_check_flags,
1392         .chipset_flush = i9xx_chipset_flush,
1393 };
1394 static const struct intel_gtt_driver sandybridge_gtt_driver = {
1395         .gen = 6,
1396         .setup = i9xx_setup,
1397         .cleanup = gen6_cleanup,
1398         .write_entry = gen6_write_entry,
1399         .dma_mask_size = 40,
1400         .check_flags = gen6_check_flags,
1401         .chipset_flush = i9xx_chipset_flush,
1402 };
1403
1404 /* Table to describe Intel GMCH and AGP/PCIE GART drivers.  At least one of
1405  * driver and gmch_driver must be non-null, and find_gmch will determine
1406  * which one should be used if a gmch_chip_id is present.
1407  */
1408 static const struct intel_gtt_driver_description {
1409         unsigned int gmch_chip_id;
1410         char *name;
1411         const struct agp_bridge_driver *gmch_driver;
1412         const struct intel_gtt_driver *gtt_driver;
1413 } intel_gtt_chipsets[] = {
1414         { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &intel_810_driver,
1415                 &i81x_gtt_driver},
1416         { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &intel_810_driver,
1417                 &i81x_gtt_driver},
1418         { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &intel_810_driver,
1419                 &i81x_gtt_driver},
1420         { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &intel_810_driver,
1421                 &i81x_gtt_driver},
1422         { PCI_DEVICE_ID_INTEL_82830_CGC, "830M",
1423                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1424         { PCI_DEVICE_ID_INTEL_82845G_IG, "830M",
1425                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1426         { PCI_DEVICE_ID_INTEL_82854_IG, "854",
1427                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1428         { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM",
1429                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1430         { PCI_DEVICE_ID_INTEL_82865_IG, "865",
1431                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1432         { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)",
1433                 &intel_fake_agp_driver, &i915_gtt_driver },
1434         { PCI_DEVICE_ID_INTEL_82915G_IG, "915G",
1435                 &intel_fake_agp_driver, &i915_gtt_driver },
1436         { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM",
1437                 &intel_fake_agp_driver, &i915_gtt_driver },
1438         { PCI_DEVICE_ID_INTEL_82945G_IG, "945G",
1439                 &intel_fake_agp_driver, &i915_gtt_driver },
1440         { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM",
1441                 &intel_fake_agp_driver, &i915_gtt_driver },
1442         { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME",
1443                 &intel_fake_agp_driver, &i915_gtt_driver },
1444         { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ",
1445                 &intel_fake_agp_driver, &i965_gtt_driver },
1446         { PCI_DEVICE_ID_INTEL_82G35_IG, "G35",
1447                 &intel_fake_agp_driver, &i965_gtt_driver },
1448         { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q",
1449                 &intel_fake_agp_driver, &i965_gtt_driver },
1450         { PCI_DEVICE_ID_INTEL_82965G_IG, "965G",
1451                 &intel_fake_agp_driver, &i965_gtt_driver },
1452         { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM",
1453                 &intel_fake_agp_driver, &i965_gtt_driver },
1454         { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE",
1455                 &intel_fake_agp_driver, &i965_gtt_driver },
1456         { PCI_DEVICE_ID_INTEL_G33_IG, "G33",
1457                 &intel_fake_agp_driver, &g33_gtt_driver },
1458         { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35",
1459                 &intel_fake_agp_driver, &g33_gtt_driver },
1460         { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33",
1461                 &intel_fake_agp_driver, &g33_gtt_driver },
1462         { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150",
1463                 &intel_fake_agp_driver, &pineview_gtt_driver },
1464         { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150",
1465                 &intel_fake_agp_driver, &pineview_gtt_driver },
1466         { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45",
1467                 &intel_fake_agp_driver, &g4x_gtt_driver },
1468         { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake",
1469                 &intel_fake_agp_driver, &g4x_gtt_driver },
1470         { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43",
1471                 &intel_fake_agp_driver, &g4x_gtt_driver },
1472         { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43",
1473                 &intel_fake_agp_driver, &g4x_gtt_driver },
1474         { PCI_DEVICE_ID_INTEL_B43_IG, "B43",
1475                 &intel_fake_agp_driver, &g4x_gtt_driver },
1476         { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43",
1477                 &intel_fake_agp_driver, &g4x_gtt_driver },
1478         { PCI_DEVICE_ID_INTEL_G41_IG, "G41",
1479                 &intel_fake_agp_driver, &g4x_gtt_driver },
1480         { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG,
1481             "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver },
1482         { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG,
1483             "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver },
1484         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG,
1485             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1486         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG,
1487             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1488         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG,
1489             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1490         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG,
1491             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1492         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG,
1493             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1494         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG,
1495             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1496         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG,
1497             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1498         { 0, NULL, NULL }
1499 };
1500
1501 static int find_gmch(u16 device)
1502 {
1503         struct pci_dev *gmch_device;
1504
1505         gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
1506         if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) {
1507                 gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL,
1508                                              device, gmch_device);
1509         }
1510
1511         if (!gmch_device)
1512                 return 0;
1513
1514         intel_private.pcidev = gmch_device;
1515         return 1;
1516 }
1517
1518 int intel_gmch_probe(struct pci_dev *pdev,
1519                                       struct agp_bridge_data *bridge)
1520 {
1521         int i, mask;
1522         bridge->driver = NULL;
1523
1524         for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) {
1525                 if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) {
1526                         bridge->driver =
1527                                 intel_gtt_chipsets[i].gmch_driver;
1528                         intel_private.driver =
1529                                 intel_gtt_chipsets[i].gtt_driver;
1530                         break;
1531                 }
1532         }
1533
1534         if (!bridge->driver)
1535                 return 0;
1536
1537         bridge->dev_private_data = &intel_private;
1538         bridge->dev = pdev;
1539
1540         intel_private.bridge_dev = pci_dev_get(pdev);
1541
1542         dev_info(&pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name);
1543
1544         mask = intel_private.driver->dma_mask_size;
1545         if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask)))
1546                 dev_err(&intel_private.pcidev->dev,
1547                         "set gfx device dma mask %d-bit failed!\n", mask);
1548         else
1549                 pci_set_consistent_dma_mask(intel_private.pcidev,
1550                                             DMA_BIT_MASK(mask));
1551
1552         if (bridge->driver == &intel_810_driver)
1553                 return 1;
1554
1555         if (intel_gtt_init() != 0)
1556                 return 0;
1557
1558         return 1;
1559 }
1560 EXPORT_SYMBOL(intel_gmch_probe);
1561
1562 const struct intel_gtt *intel_gtt_get(void)
1563 {
1564         return &intel_private.base;
1565 }
1566 EXPORT_SYMBOL(intel_gtt_get);
1567
1568 void intel_gmch_remove(struct pci_dev *pdev)
1569 {
1570         if (intel_private.pcidev)
1571                 pci_dev_put(intel_private.pcidev);
1572         if (intel_private.bridge_dev)
1573                 pci_dev_put(intel_private.bridge_dev);
1574 }
1575 EXPORT_SYMBOL(intel_gmch_remove);
1576
1577 MODULE_AUTHOR("Dave Jones <davej@redhat.com>");
1578 MODULE_LICENSE("GPL and additional rights");