drivers/char/agp/intel-gtt.c

   1 /*
   2  * Intel GTT (Graphics Translation Table) routines
   3  *
   4  * Caveat: This driver implements the linux agp interface, but this is far from
   5  * a agp driver! GTT support ended up here for purely historical reasons: The
   6  * old userspace intel graphics drivers needed an interface to map memory into
   7  * the GTT. And the drm provides a default interface for graphic devices sitting
   8  * on an agp port. So it made sense to fake the GTT support as an agp port to
   9  * avoid having to create a new api.
  10  *
  11  * With gem this does not make much sense anymore, just needlessly complicates
  12  * the code. But as long as the old graphics stack is still support, it's stuck
  13  * here.
  14  *
  15  * /fairy-tale-mode off
  16  */
  17
  18 #include <linux/module.h>
  19 #include <linux/pci.h>
  20 #include <linux/init.h>
  21 #include <linux/kernel.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/agp_backend.h>
  24 #include <asm/smp.h>
  25 #include "agp.h"
  26 #include "intel-agp.h"
  27 #include <linux/intel-gtt.h>
  28 #include <drm/intel-gtt.h>
  29
  30 /*
  31  * If we have Intel graphics, we're not going to have anything other than
  32  * an Intel IOMMU. So make the correct use of the PCI DMA API contingent
  33  * on the Intel IOMMU support (CONFIG_DMAR).
  34  * Only newer chipsets need to bother with this, of course.
  35  */
  36 #ifdef CONFIG_DMAR
  37 #define USE_PCI_DMA_API 1
  38 #else
  39 #define USE_PCI_DMA_API 0
  40 #endif
  41
  42 /* Max amount of stolen space, anything above will be returned to Linux */
  43 int intel_max_stolen = 32 * 1024 * 1024;
  44
  45 static const struct aper_size_info_fixed intel_i810_sizes[] =
  46 {
  47         {64, 16384, 4},
  48         /* The 32M mode still requires a 64k gatt */
  49         {32, 8192, 4}
  50 };
  51
  52 #define AGP_DCACHE_MEMORY       1
  53 #define AGP_PHYS_MEMORY         2
  54 #define INTEL_AGP_CACHED_MEMORY 3
  55
  56 static struct gatt_mask intel_i810_masks[] =
  57 {
  58         {.mask = I810_PTE_VALID, .type = 0},
  59         {.mask = (I810_PTE_VALID | I810_PTE_LOCAL), .type = AGP_DCACHE_MEMORY},
  60         {.mask = I810_PTE_VALID, .type = 0},
  61         {.mask = I810_PTE_VALID | I830_PTE_SYSTEM_CACHED,
  62          .type = INTEL_AGP_CACHED_MEMORY}
  63 };
  64
  65 #define INTEL_AGP_UNCACHED_MEMORY              0
  66 #define INTEL_AGP_CACHED_MEMORY_LLC            1
  67 #define INTEL_AGP_CACHED_MEMORY_LLC_GFDT       2
  68 #define INTEL_AGP_CACHED_MEMORY_LLC_MLC        3
  69 #define INTEL_AGP_CACHED_MEMORY_LLC_MLC_GFDT   4
  70
  71 struct intel_gtt_driver {
  72         unsigned int gen : 8;
  73         unsigned int is_g33 : 1;
  74         unsigned int is_pineview : 1;
  75         unsigned int is_ironlake : 1;
  76         unsigned int dma_mask_size : 8;
  77         /* Chipset specific GTT setup */
  78         int (*setup)(void);
  79         /* This should undo anything done in ->setup() save the unmapping
  80          * of the mmio register file, that's done in the generic code. */
  81         void (*cleanup)(void);
  82         void (*write_entry)(dma_addr_t addr, unsigned int entry, unsigned int flags);
  83         /* Flags is a more or less chipset specific opaque value.
  84          * For chipsets that need to support old ums (non-gem) code, this
  85          * needs to be identical to the various supported agp memory types! */
  86         bool (*check_flags)(unsigned int flags);
  87         void (*chipset_flush)(void);
  88 };
  89
  90 static struct _intel_private {
  91         struct intel_gtt base;
  92         const struct intel_gtt_driver *driver;
  93         struct pci_dev *pcidev; /* device one */
  94         struct pci_dev *bridge_dev;
  95         u8 __iomem *registers;
  96         phys_addr_t gtt_bus_addr;
  97         phys_addr_t gma_bus_addr;
  98         phys_addr_t pte_bus_addr;
  99         u32 __iomem *gtt;               /* I915G */
 100         int num_dcache_entries;
 101         union {
 102                 void __iomem *i9xx_flush_page;
 103                 void *i8xx_flush_page;
 104         };
 105         struct page *i8xx_page;
 106         struct resource ifp_resource;
 107         int resource_valid;
 108         struct page *scratch_page;
 109         dma_addr_t scratch_page_dma;
 110 } intel_private;
 111
 112 #define INTEL_GTT_GEN   intel_private.driver->gen
 113 #define IS_G33          intel_private.driver->is_g33
 114 #define IS_PINEVIEW     intel_private.driver->is_pineview
 115 #define IS_IRONLAKE     intel_private.driver->is_ironlake
 116
 117 static void intel_agp_free_sglist(struct agp_memory *mem)
 118 {
 119         struct sg_table st;
 120
 121         st.sgl = mem->sg_list;
 122         st.orig_nents = st.nents = mem->page_count;
 123
 124         sg_free_table(&st);
 125
 126         mem->sg_list = NULL;
 127         mem->num_sg = 0;
 128 }
 129
 130 static int intel_agp_map_memory(struct agp_memory *mem)
 131 {
 132         struct sg_table st;
 133         struct scatterlist *sg;
 134         int i;
 135
 136         if (mem->sg_list)
 137                 return 0; /* already mapped (for e.g. resume */
 138
 139         DBG("try mapping %lu pages\n", (unsigned long)mem->page_count);
 140
 141         if (sg_alloc_table(&st, mem->page_count, GFP_KERNEL))
 142                 goto err;
 143
 144         mem->sg_list = sg = st.sgl;
 145
 146         for (i = 0 ; i < mem->page_count; i++, sg = sg_next(sg))
 147                 sg_set_page(sg, mem->pages[i], PAGE_SIZE, 0);
 148
 149         mem->num_sg = pci_map_sg(intel_private.pcidev, mem->sg_list,
 150                                  mem->page_count, PCI_DMA_BIDIRECTIONAL);
 151         if (unlikely(!mem->num_sg))
 152                 goto err;
 153
 154         return 0;
 155
 156 err:
 157         sg_free_table(&st);
 158         return -ENOMEM;
 159 }
 160
 161 static void intel_agp_unmap_memory(struct agp_memory *mem)
 162 {
 163         DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);
 164
 165         pci_unmap_sg(intel_private.pcidev, mem->sg_list,
 166                      mem->page_count, PCI_DMA_BIDIRECTIONAL);
 167         intel_agp_free_sglist(mem);
 168 }
 169
 170 static int intel_i810_fetch_size(void)
 171 {
 172         u32 smram_miscc;
 173         struct aper_size_info_fixed *values;
 174
 175         pci_read_config_dword(intel_private.bridge_dev,
 176                               I810_SMRAM_MISCC, &smram_miscc);
 177         values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
 178
 179         if ((smram_miscc & I810_GMS) == I810_GMS_DISABLE) {
 180                 dev_warn(&intel_private.bridge_dev->dev, "i810 is disabled\n");
 181                 return 0;
 182         }
 183         if ((smram_miscc & I810_GFX_MEM_WIN_SIZE) == I810_GFX_MEM_WIN_32M) {
 184                 agp_bridge->current_size = (void *) (values + 1);
 185                 agp_bridge->aperture_size_idx = 1;
 186                 return values[1].size;
 187         } else {
 188                 agp_bridge->current_size = (void *) (values);
 189                 agp_bridge->aperture_size_idx = 0;
 190                 return values[0].size;
 191         }
 192
 193         return 0;
 194 }
 195
 196 static int intel_i810_configure(void)
 197 {
 198         struct aper_size_info_fixed *current_size;
 199         u32 temp;
 200         int i;
 201
 202         current_size = A_SIZE_FIX(agp_bridge->current_size);
 203
 204         if (!intel_private.registers) {
 205                 pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &temp);
 206                 temp &= 0xfff80000;
 207
 208                 intel_private.registers = ioremap(temp, 128 * 4096);
 209                 if (!intel_private.registers) {
 210                         dev_err(&intel_private.pcidev->dev,
 211                                 "can't remap memory\n");
 212                         return -ENOMEM;
 213                 }
 214         }
 215
 216         if ((readl(intel_private.registers+I810_DRAM_CTL)
 217                 & I810_DRAM_ROW_0) == I810_DRAM_ROW_0_SDRAM) {
 218                 /* This will need to be dynamically assigned */
 219                 dev_info(&intel_private.pcidev->dev,
 220                          "detected 4MB dedicated video ram\n");
 221                 intel_private.num_dcache_entries = 1024;
 222         }
 223         pci_read_config_dword(intel_private.pcidev, I810_GMADDR, &temp);
 224         agp_bridge->gart_bus_addr = (temp & PCI_BASE_ADDRESS_MEM_MASK);
 225         writel(agp_bridge->gatt_bus_addr | I810_PGETBL_ENABLED, intel_private.registers+I810_PGETBL_CTL);
 226         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
 227
 228         if (agp_bridge->driver->needs_scratch_page) {
 229                 for (i = 0; i < current_size->num_entries; i++) {
 230                         writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 231                 }
 232                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4)); /* PCI posting. */
 233         }
 234         global_cache_flush();
 235         return 0;
 236 }
 237
 238 static void intel_i810_cleanup(void)
 239 {
 240         writel(0, intel_private.registers+I810_PGETBL_CTL);
 241         readl(intel_private.registers); /* PCI Posting. */
 242         iounmap(intel_private.registers);
 243 }
 244
 245 static void intel_fake_agp_enable(struct agp_bridge_data *bridge, u32 mode)
 246 {
 247         return;
 248 }
 249
 250 /* Exists to support ARGB cursors */
 251 static struct page *i8xx_alloc_pages(void)
 252 {
 253         struct page *page;
 254
 255         page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2);
 256         if (page == NULL)
 257                 return NULL;
 258
 259         if (set_pages_uc(page, 4) < 0) {
 260                 set_pages_wb(page, 4);
 261                 __free_pages(page, 2);
 262                 return NULL;
 263         }
 264         get_page(page);
 265         atomic_inc(&agp_bridge->current_memory_agp);
 266         return page;
 267 }
 268
 269 static void i8xx_destroy_pages(struct page *page)
 270 {
 271         if (page == NULL)
 272                 return;
 273
 274         set_pages_wb(page, 4);
 275         put_page(page);
 276         __free_pages(page, 2);
 277         atomic_dec(&agp_bridge->current_memory_agp);
 278 }
 279
 280 static int intel_i810_insert_entries(struct agp_memory *mem, off_t pg_start,
 281                                 int type)
 282 {
 283         int i, j, num_entries;
 284         void *temp;
 285         int ret = -EINVAL;
 286         int mask_type;
 287
 288         if (mem->page_count == 0)
 289                 goto out;
 290
 291         temp = agp_bridge->current_size;
 292         num_entries = A_SIZE_FIX(temp)->num_entries;
 293
 294         if ((pg_start + mem->page_count) > num_entries)
 295                 goto out_err;
 296
 297
 298         for (j = pg_start; j < (pg_start + mem->page_count); j++) {
 299                 if (!PGE_EMPTY(agp_bridge, readl(agp_bridge->gatt_table+j))) {
 300                         ret = -EBUSY;
 301                         goto out_err;
 302                 }
 303         }
 304
 305         if (type != mem->type)
 306                 goto out_err;
 307
 308         mask_type = agp_bridge->driver->agp_type_to_mask_type(agp_bridge, type);
 309
 310         switch (mask_type) {
 311         case AGP_DCACHE_MEMORY:
 312                 if (!mem->is_flushed)
 313                         global_cache_flush();
 314                 for (i = pg_start; i < (pg_start + mem->page_count); i++) {
 315                         writel((i*4096)|I810_PTE_LOCAL|I810_PTE_VALID,
 316                                intel_private.registers+I810_PTE_BASE+(i*4));
 317                 }
 318                 readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
 319                 break;
 320         case AGP_PHYS_MEMORY:
 321         case AGP_NORMAL_MEMORY:
 322                 if (!mem->is_flushed)
 323                         global_cache_flush();
 324                 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
 325                         writel(agp_bridge->driver->mask_memory(agp_bridge,
 326                                         page_to_phys(mem->pages[i]), mask_type),
 327                                intel_private.registers+I810_PTE_BASE+(j*4));
 328                 }
 329                 readl(intel_private.registers+I810_PTE_BASE+((j-1)*4));
 330                 break;
 331         default:
 332                 goto out_err;
 333         }
 334
 335 out:
 336         ret = 0;
 337 out_err:
 338         mem->is_flushed = true;
 339         return ret;
 340 }
 341
 342 static int intel_i810_remove_entries(struct agp_memory *mem, off_t pg_start,
 343                                 int type)
 344 {
 345         int i;
 346
 347         if (mem->page_count == 0)
 348                 return 0;
 349
 350         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
 351                 writel(agp_bridge->scratch_page, intel_private.registers+I810_PTE_BASE+(i*4));
 352         }
 353         readl(intel_private.registers+I810_PTE_BASE+((i-1)*4));
 354
 355         return 0;
 356 }
 357
 358 /*
 359  * The i810/i830 requires a physical address to program its mouse
 360  * pointer into hardware.
 361  * However the Xserver still writes to it through the agp aperture.
 362  */
 363 static struct agp_memory *alloc_agpphysmem_i8xx(size_t pg_count, int type)
 364 {
 365         struct agp_memory *new;
 366         struct page *page;
 367
 368         switch (pg_count) {
 369         case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge);
 370                 break;
 371         case 4:
 372                 /* kludge to get 4 physical pages for ARGB cursor */
 373                 page = i8xx_alloc_pages();
 374                 break;
 375         default:
 376                 return NULL;
 377         }
 378
 379         if (page == NULL)
 380                 return NULL;
 381
 382         new = agp_create_memory(pg_count);
 383         if (new == NULL)
 384                 return NULL;
 385
 386         new->pages[0] = page;
 387         if (pg_count == 4) {
 388                 /* kludge to get 4 physical pages for ARGB cursor */
 389                 new->pages[1] = new->pages[0] + 1;
 390                 new->pages[2] = new->pages[1] + 1;
 391                 new->pages[3] = new->pages[2] + 1;
 392         }
 393         new->page_count = pg_count;
 394         new->num_scratch_pages = pg_count;
 395         new->type = AGP_PHYS_MEMORY;
 396         new->physical = page_to_phys(new->pages[0]);
 397         return new;
 398 }
 399
 400 static struct agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type)
 401 {
 402         struct agp_memory *new;
 403
 404         if (type == AGP_DCACHE_MEMORY) {
 405                 if (pg_count != intel_private.num_dcache_entries)
 406                         return NULL;
 407
 408                 new = agp_create_memory(1);
 409                 if (new == NULL)
 410                         return NULL;
 411
 412                 new->type = AGP_DCACHE_MEMORY;
 413                 new->page_count = pg_count;
 414                 new->num_scratch_pages = 0;
 415                 agp_free_page_array(new);
 416                 return new;
 417         }
 418         if (type == AGP_PHYS_MEMORY)
 419                 return alloc_agpphysmem_i8xx(pg_count, type);
 420         return NULL;
 421 }
 422
 423 static void intel_i810_free_by_type(struct agp_memory *curr)
 424 {
 425         agp_free_key(curr->key);
 426         if (curr->type == AGP_PHYS_MEMORY) {
 427                 if (curr->page_count == 4)
 428                         i8xx_destroy_pages(curr->pages[0]);
 429                 else {
 430                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 431                                                              AGP_PAGE_DESTROY_UNMAP);
 432                         agp_bridge->driver->agp_destroy_page(curr->pages[0],
 433                                                              AGP_PAGE_DESTROY_FREE);
 434                 }
 435                 agp_free_page_array(curr);
 436         }
 437         kfree(curr);
 438 }
 439
 440 static unsigned long intel_i810_mask_memory(struct agp_bridge_data *bridge,
 441                                             dma_addr_t addr, int type)
 442 {
 443         /* Type checking must be done elsewhere */
 444         return addr | bridge->driver->masks[type].mask;
 445 }
 446
 447 static int intel_gtt_setup_scratch_page(void)
 448 {
 449         struct page *page;
 450         dma_addr_t dma_addr;
 451
 452         page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
 453         if (page == NULL)
 454                 return -ENOMEM;
 455         get_page(page);
 456         set_pages_uc(page, 1);
 457
 458         if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
 459                 dma_addr = pci_map_page(intel_private.pcidev, page, 0,
 460                                     PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 461                 if (pci_dma_mapping_error(intel_private.pcidev, dma_addr))
 462                         return -EINVAL;
 463
 464                 intel_private.scratch_page_dma = dma_addr;
 465         } else
 466                 intel_private.scratch_page_dma = page_to_phys(page);
 467
 468         intel_private.scratch_page = page;
 469
 470         return 0;
 471 }
 472
 473 static const struct aper_size_info_fixed const intel_fake_agp_sizes[] = {
 474         {128, 32768, 5},
 475         /* The 64M mode still requires a 128k gatt */
 476         {64, 16384, 5},
 477         {256, 65536, 6},
 478         {512, 131072, 7},
 479 };
 480
 481 static unsigned int intel_gtt_stolen_entries(void)
 482 {
 483         u16 gmch_ctrl;
 484         u8 rdct;
 485         int local = 0;
 486         static const int ddt[4] = { 0, 16, 32, 64 };
 487         unsigned int overhead_entries, stolen_entries;
 488         unsigned int stolen_size = 0;
 489
 490         pci_read_config_word(intel_private.bridge_dev,
 491                              I830_GMCH_CTRL, &gmch_ctrl);
 492
 493         if (INTEL_GTT_GEN > 4 || IS_PINEVIEW)
 494                 overhead_entries = 0;
 495         else
 496                 overhead_entries = intel_private.base.gtt_mappable_entries
 497                         / 1024;
 498
 499         overhead_entries += 1; /* BIOS popup */
 500
 501         if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
 502             intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
 503                 switch (gmch_ctrl & I830_GMCH_GMS_MASK) {
 504                 case I830_GMCH_GMS_STOLEN_512:
 505                         stolen_size = KB(512);
 506                         break;
 507                 case I830_GMCH_GMS_STOLEN_1024:
 508                         stolen_size = MB(1);
 509                         break;
 510                 case I830_GMCH_GMS_STOLEN_8192:
 511                         stolen_size = MB(8);
 512                         break;
 513                 case I830_GMCH_GMS_LOCAL:
 514                         rdct = readb(intel_private.registers+I830_RDRAM_CHANNEL_TYPE);
 515                         stolen_size = (I830_RDRAM_ND(rdct) + 1) *
 516                                         MB(ddt[I830_RDRAM_DDT(rdct)]);
 517                         local = 1;
 518                         break;
 519                 default:
 520                         stolen_size = 0;
 521                         break;
 522                 }
 523         } else if (INTEL_GTT_GEN == 6) {
 524                 /*
 525                  * SandyBridge has new memory control reg at 0x50.w
 526                  */
 527                 u16 snb_gmch_ctl;
 528                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 529                 switch (snb_gmch_ctl & SNB_GMCH_GMS_STOLEN_MASK) {
 530                 case SNB_GMCH_GMS_STOLEN_32M:
 531                         stolen_size = MB(32);
 532                         break;
 533                 case SNB_GMCH_GMS_STOLEN_64M:
 534                         stolen_size = MB(64);
 535                         break;
 536                 case SNB_GMCH_GMS_STOLEN_96M:
 537                         stolen_size = MB(96);
 538                         break;
 539                 case SNB_GMCH_GMS_STOLEN_128M:
 540                         stolen_size = MB(128);
 541                         break;
 542                 case SNB_GMCH_GMS_STOLEN_160M:
 543                         stolen_size = MB(160);
 544                         break;
 545                 case SNB_GMCH_GMS_STOLEN_192M:
 546                         stolen_size = MB(192);
 547                         break;
 548                 case SNB_GMCH_GMS_STOLEN_224M:
 549                         stolen_size = MB(224);
 550                         break;
 551                 case SNB_GMCH_GMS_STOLEN_256M:
 552                         stolen_size = MB(256);
 553                         break;
 554                 case SNB_GMCH_GMS_STOLEN_288M:
 555                         stolen_size = MB(288);
 556                         break;
 557                 case SNB_GMCH_GMS_STOLEN_320M:
 558                         stolen_size = MB(320);
 559                         break;
 560                 case SNB_GMCH_GMS_STOLEN_352M:
 561                         stolen_size = MB(352);
 562                         break;
 563                 case SNB_GMCH_GMS_STOLEN_384M:
 564                         stolen_size = MB(384);
 565                         break;
 566                 case SNB_GMCH_GMS_STOLEN_416M:
 567                         stolen_size = MB(416);
 568                         break;
 569                 case SNB_GMCH_GMS_STOLEN_448M:
 570                         stolen_size = MB(448);
 571                         break;
 572                 case SNB_GMCH_GMS_STOLEN_480M:
 573                         stolen_size = MB(480);
 574                         break;
 575                 case SNB_GMCH_GMS_STOLEN_512M:
 576                         stolen_size = MB(512);
 577                         break;
 578                 }
 579         } else {
 580                 switch (gmch_ctrl & I855_GMCH_GMS_MASK) {
 581                 case I855_GMCH_GMS_STOLEN_1M:
 582                         stolen_size = MB(1);
 583                         break;
 584                 case I855_GMCH_GMS_STOLEN_4M:
 585                         stolen_size = MB(4);
 586                         break;
 587                 case I855_GMCH_GMS_STOLEN_8M:
 588                         stolen_size = MB(8);
 589                         break;
 590                 case I855_GMCH_GMS_STOLEN_16M:
 591                         stolen_size = MB(16);
 592                         break;
 593                 case I855_GMCH_GMS_STOLEN_32M:
 594                         stolen_size = MB(32);
 595                         break;
 596                 case I915_GMCH_GMS_STOLEN_48M:
 597                         stolen_size = MB(48);
 598                         break;
 599                 case I915_GMCH_GMS_STOLEN_64M:
 600                         stolen_size = MB(64);
 601                         break;
 602                 case G33_GMCH_GMS_STOLEN_128M:
 603                         stolen_size = MB(128);
 604                         break;
 605                 case G33_GMCH_GMS_STOLEN_256M:
 606                         stolen_size = MB(256);
 607                         break;
 608                 case INTEL_GMCH_GMS_STOLEN_96M:
 609                         stolen_size = MB(96);
 610                         break;
 611                 case INTEL_GMCH_GMS_STOLEN_160M:
 612                         stolen_size = MB(160);
 613                         break;
 614                 case INTEL_GMCH_GMS_STOLEN_224M:
 615                         stolen_size = MB(224);
 616                         break;
 617                 case INTEL_GMCH_GMS_STOLEN_352M:
 618                         stolen_size = MB(352);
 619                         break;
 620                 default:
 621                         stolen_size = 0;
 622                         break;
 623                 }
 624         }
 625
 626         if (!local && stolen_size > intel_max_stolen) {
 627                 dev_info(&intel_private.bridge_dev->dev,
 628                          "detected %dK stolen memory, trimming to %dK\n",
 629                          stolen_size / KB(1), intel_max_stolen / KB(1));
 630                 stolen_size = intel_max_stolen;
 631         } else if (stolen_size > 0) {
 632                 dev_info(&intel_private.bridge_dev->dev, "detected %dK %s memory\n",
 633                        stolen_size / KB(1), local ? "local" : "stolen");
 634         } else {
 635                 dev_info(&intel_private.bridge_dev->dev,
 636                        "no pre-allocated video memory detected\n");
 637                 stolen_size = 0;
 638         }
 639
 640         stolen_entries = stolen_size/KB(4) - overhead_entries;
 641
 642         return stolen_entries;
 643 }
 644
 645 static unsigned int intel_gtt_total_entries(void)
 646 {
 647         int size;
 648
 649         if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5) {
 650                 u32 pgetbl_ctl;
 651                 pgetbl_ctl = readl(intel_private.registers+I810_PGETBL_CTL);
 652
 653                 switch (pgetbl_ctl & I965_PGETBL_SIZE_MASK) {
 654                 case I965_PGETBL_SIZE_128KB:
 655                         size = KB(128);
 656                         break;
 657                 case I965_PGETBL_SIZE_256KB:
 658                         size = KB(256);
 659                         break;
 660                 case I965_PGETBL_SIZE_512KB:
 661                         size = KB(512);
 662                         break;
 663                 case I965_PGETBL_SIZE_1MB:
 664                         size = KB(1024);
 665                         break;
 666                 case I965_PGETBL_SIZE_2MB:
 667                         size = KB(2048);
 668                         break;
 669                 case I965_PGETBL_SIZE_1_5MB:
 670                         size = KB(1024 + 512);
 671                         break;
 672                 default:
 673                         dev_info(&intel_private.pcidev->dev,
 674                                  "unknown page table size, assuming 512KB\n");
 675                         size = KB(512);
 676                 }
 677
 678                 return size/4;
 679         } else if (INTEL_GTT_GEN == 6) {
 680                 u16 snb_gmch_ctl;
 681
 682                 pci_read_config_word(intel_private.pcidev, SNB_GMCH_CTRL, &snb_gmch_ctl);
 683                 switch (snb_gmch_ctl & SNB_GTT_SIZE_MASK) {
 684                 default:
 685                 case SNB_GTT_SIZE_0M:
 686                         printk(KERN_ERR "Bad GTT size mask: 0x%04x.\n", snb_gmch_ctl);
 687                         size = MB(0);
 688                         break;
 689                 case SNB_GTT_SIZE_1M:
 690                         size = MB(1);
 691                         break;
 692                 case SNB_GTT_SIZE_2M:
 693                         size = MB(2);
 694                         break;
 695                 }
 696                 return size/4;
 697         } else {
 698                 /* On previous hardware, the GTT size was just what was
 699                  * required to map the aperture.
 700                  */
 701                 return intel_private.base.gtt_mappable_entries;
 702         }
 703 }
 704
 705 static unsigned int intel_gtt_mappable_entries(void)
 706 {
 707         unsigned int aperture_size;
 708
 709         if (INTEL_GTT_GEN == 2) {
 710                 u16 gmch_ctrl;
 711
 712                 pci_read_config_word(intel_private.bridge_dev,
 713                                      I830_GMCH_CTRL, &gmch_ctrl);
 714
 715                 if ((gmch_ctrl & I830_GMCH_MEM_MASK) == I830_GMCH_MEM_64M)
 716                         aperture_size = MB(64);
 717                 else
 718                         aperture_size = MB(128);
 719         } else {
 720                 /* 9xx supports large sizes, just look at the length */
 721                 aperture_size = pci_resource_len(intel_private.pcidev, 2);
 722         }
 723
 724         return aperture_size >> PAGE_SHIFT;
 725 }
 726
 727 static void intel_gtt_teardown_scratch_page(void)
 728 {
 729         set_pages_wb(intel_private.scratch_page, 1);
 730         pci_unmap_page(intel_private.pcidev, intel_private.scratch_page_dma,
 731                        PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 732         put_page(intel_private.scratch_page);
 733         __free_page(intel_private.scratch_page);
 734 }
 735
 736 static void intel_gtt_cleanup(void)
 737 {
 738         intel_private.driver->cleanup();
 739
 740         iounmap(intel_private.gtt);
 741         iounmap(intel_private.registers);
 742
 743         intel_gtt_teardown_scratch_page();
 744 }
 745
 746 static int intel_gtt_init(void)
 747 {
 748         u32 gtt_map_size;
 749         int ret;
 750
 751         ret = intel_private.driver->setup();
 752         if (ret != 0)
 753                 return ret;
 754
 755         intel_private.base.gtt_mappable_entries = intel_gtt_mappable_entries();
 756         intel_private.base.gtt_total_entries = intel_gtt_total_entries();
 757
 758         dev_info(&intel_private.bridge_dev->dev,
 759                         "detected gtt size: %dK total, %dK mappable\n",
 760                         intel_private.base.gtt_total_entries * 4,
 761                         intel_private.base.gtt_mappable_entries * 4);
 762
 763         gtt_map_size = intel_private.base.gtt_total_entries * 4;
 764
 765         intel_private.gtt = ioremap(intel_private.gtt_bus_addr,
 766                                     gtt_map_size);
 767         if (!intel_private.gtt) {
 768                 intel_private.driver->cleanup();
 769                 iounmap(intel_private.registers);
 770                 return -ENOMEM;
 771         }
 772
 773         global_cache_flush();   /* FIXME: ? */
 774
 775         /* we have to call this as early as possible after the MMIO base address is known */
 776         intel_private.base.gtt_stolen_entries = intel_gtt_stolen_entries();
 777         if (intel_private.base.gtt_stolen_entries == 0) {
 778                 intel_private.driver->cleanup();
 779                 iounmap(intel_private.registers);
 780                 iounmap(intel_private.gtt);
 781                 return -ENOMEM;
 782         }
 783
 784         ret = intel_gtt_setup_scratch_page();
 785         if (ret != 0) {
 786                 intel_gtt_cleanup();
 787                 return ret;
 788         }
 789
 790         return 0;
 791 }
 792
 793 static int intel_fake_agp_fetch_size(void)
 794 {
 795         int num_sizes = ARRAY_SIZE(intel_fake_agp_sizes);
 796         unsigned int aper_size;
 797         int i;
 798
 799         aper_size = (intel_private.base.gtt_mappable_entries << PAGE_SHIFT)
 800                     / MB(1);
 801
 802         for (i = 0; i < num_sizes; i++) {
 803                 if (aper_size == intel_fake_agp_sizes[i].size) {
 804                         agp_bridge->current_size =
 805                                 (void *) (intel_fake_agp_sizes + i);
 806                         return aper_size;
 807                 }
 808         }
 809
 810         return 0;
 811 }
 812
 813 static void i830_cleanup(void)
 814 {
 815         kunmap(intel_private.i8xx_page);
 816         intel_private.i8xx_flush_page = NULL;
 817
 818         __free_page(intel_private.i8xx_page);
 819         intel_private.i8xx_page = NULL;
 820 }
 821
 822 static void intel_i830_setup_flush(void)
 823 {
 824         /* return if we've already set the flush mechanism up */
 825         if (intel_private.i8xx_page)
 826                 return;
 827
 828         intel_private.i8xx_page = alloc_page(GFP_KERNEL);
 829         if (!intel_private.i8xx_page)
 830                 return;
 831
 832         intel_private.i8xx_flush_page = kmap(intel_private.i8xx_page);
 833         if (!intel_private.i8xx_flush_page)
 834                 i830_cleanup();
 835 }
 836
 837 /* The chipset_flush interface needs to get data that has already been
 838  * flushed out of the CPU all the way out to main memory, because the GPU
 839  * doesn't snoop those buffers.
 840  *
 841  * The 8xx series doesn't have the same lovely interface for flushing the
 842  * chipset write buffers that the later chips do. According to the 865
 843  * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
 844  * that buffer out, we just fill 1KB and clflush it out, on the assumption
 845  * that it'll push whatever was in there out.  It appears to work.
 846  */
 847 static void i830_chipset_flush(void)
 848 {
 849         unsigned int *pg = intel_private.i8xx_flush_page;
 850
 851         memset(pg, 0, 1024);
 852
 853         if (cpu_has_clflush)
 854                 clflush_cache_range(pg, 1024);
 855         else if (wbinvd_on_all_cpus() != 0)
 856                 printk(KERN_ERR "Timed out waiting for cache flush.\n");
 857 }
 858
 859 static void i830_write_entry(dma_addr_t addr, unsigned int entry,
 860                              unsigned int flags)
 861 {
 862         u32 pte_flags = I810_PTE_VALID;
 863
 864         switch (flags) {
 865         case AGP_DCACHE_MEMORY:
 866                 pte_flags |= I810_PTE_LOCAL;
 867                 break;
 868         case AGP_USER_CACHED_MEMORY:
 869                 pte_flags |= I830_PTE_SYSTEM_CACHED;
 870                 break;
 871         }
 872
 873         writel(addr | pte_flags, intel_private.gtt + entry);
 874 }
 875
 876 static void intel_enable_gtt(void)
 877 {
 878         u32 gma_addr;
 879         u16 gmch_ctrl;
 880
 881         if (INTEL_GTT_GEN == 2)
 882                 pci_read_config_dword(intel_private.pcidev, I810_GMADDR,
 883                                       &gma_addr);
 884         else
 885                 pci_read_config_dword(intel_private.pcidev, I915_GMADDR,
 886                                       &gma_addr);
 887
 888         intel_private.gma_bus_addr = (gma_addr & PCI_BASE_ADDRESS_MEM_MASK);
 889
 890         pci_read_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, &gmch_ctrl);
 891         gmch_ctrl |= I830_GMCH_ENABLED;
 892         pci_write_config_word(intel_private.bridge_dev, I830_GMCH_CTRL, gmch_ctrl);
 893
 894         writel(intel_private.pte_bus_addr|I810_PGETBL_ENABLED,
 895                intel_private.registers+I810_PGETBL_CTL);
 896         readl(intel_private.registers+I810_PGETBL_CTL); /* PCI Posting. */
 897 }
 898
 899 static int i830_setup(void)
 900 {
 901         u32 reg_addr;
 902
 903         pci_read_config_dword(intel_private.pcidev, I810_MMADDR, &reg_addr);
 904         reg_addr &= 0xfff80000;
 905
 906         intel_private.registers = ioremap(reg_addr, KB(64));
 907         if (!intel_private.registers)
 908                 return -ENOMEM;
 909
 910         intel_private.gtt_bus_addr = reg_addr + I810_PTE_BASE;
 911         intel_private.pte_bus_addr =
 912                 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
 913
 914         intel_i830_setup_flush();
 915
 916         return 0;
 917 }
 918
 919 static int intel_fake_agp_create_gatt_table(struct agp_bridge_data *bridge)
 920 {
 921         agp_bridge->gatt_table_real = NULL;
 922         agp_bridge->gatt_table = NULL;
 923         agp_bridge->gatt_bus_addr = 0;
 924
 925         return 0;
 926 }
 927
 928 static int intel_fake_agp_free_gatt_table(struct agp_bridge_data *bridge)
 929 {
 930         return 0;
 931 }
 932
 933 static int intel_fake_agp_configure(void)
 934 {
 935         int i;
 936
 937         intel_enable_gtt();
 938
 939         agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;
 940
 941         for (i = intel_private.base.gtt_stolen_entries;
 942                         i < intel_private.base.gtt_total_entries; i++) {
 943                 intel_private.driver->write_entry(intel_private.scratch_page_dma,
 944                                                   i, 0);
 945         }
 946         readl(intel_private.gtt+i-1);   /* PCI Posting. */
 947
 948         global_cache_flush();
 949
 950         return 0;
 951 }
 952
 953 static bool i830_check_flags(unsigned int flags)
 954 {
 955         switch (flags) {
 956         case 0:
 957         case AGP_PHYS_MEMORY:
 958         case AGP_USER_CACHED_MEMORY:
 959         case AGP_USER_MEMORY:
 960                 return true;
 961         }
 962
 963         return false;
 964 }
 965
 966 static void intel_gtt_insert_sg_entries(struct scatterlist *sg_list,
 967                                         unsigned int sg_len,
 968                                         unsigned int pg_start,
 969                                         unsigned int flags)
 970 {
 971         struct scatterlist *sg;
 972         unsigned int len, m;
 973         int i, j;
 974
 975         j = pg_start;
 976
 977         /* sg may merge pages, but we have to separate
 978          * per-page addr for GTT */
 979         for_each_sg(sg_list, sg, sg_len, i) {
 980                 len = sg_dma_len(sg) >> PAGE_SHIFT;
 981                 for (m = 0; m < len; m++) {
 982                         dma_addr_t addr = sg_dma_address(sg) + (m << PAGE_SHIFT);
 983                         intel_private.driver->write_entry(addr,
 984                                                           j, flags);
 985                         j++;
 986                 }
 987         }
 988         readl(intel_private.gtt+j-1);
 989 }
 990
 991 static int intel_fake_agp_insert_entries(struct agp_memory *mem,
 992                                          off_t pg_start, int type)
 993 {
 994         int i, j;
 995         int ret = -EINVAL;
 996
 997         if (mem->page_count == 0)
 998                 goto out;
 999
1000         if (pg_start < intel_private.base.gtt_stolen_entries) {
1001                 dev_printk(KERN_DEBUG, &intel_private.pcidev->dev,
1002                            "pg_start == 0x%.8lx, gtt_stolen_entries == 0x%.8x\n",
1003                            pg_start, intel_private.base.gtt_stolen_entries);
1004
1005                 dev_info(&intel_private.pcidev->dev,
1006                          "trying to insert into local/stolen memory\n");
1007                 goto out_err;
1008         }
1009
1010         if ((pg_start + mem->page_count) > intel_private.base.gtt_total_entries)
1011                 goto out_err;
1012
1013         if (type != mem->type)
1014                 goto out_err;
1015
1016         if (!intel_private.driver->check_flags(type))
1017                 goto out_err;
1018
1019         if (!mem->is_flushed)
1020                 global_cache_flush();
1021
1022         if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2) {
1023                 ret = intel_agp_map_memory(mem);
1024                 if (ret != 0)
1025                         return ret;
1026
1027                 intel_gtt_insert_sg_entries(mem->sg_list, mem->num_sg,
1028                                             pg_start, type);
1029         } else {
1030                 for (i = 0, j = pg_start; i < mem->page_count; i++, j++) {
1031                         dma_addr_t addr = page_to_phys(mem->pages[i]);
1032                         intel_private.driver->write_entry(addr,
1033                                                           j, type);
1034                 }
1035                 readl(intel_private.gtt+j-1);
1036         }
1037
1038 out:
1039         ret = 0;
1040 out_err:
1041         mem->is_flushed = true;
1042         return ret;
1043 }
1044
1045 static int intel_fake_agp_remove_entries(struct agp_memory *mem,
1046                                          off_t pg_start, int type)
1047 {
1048         int i;
1049
1050         if (mem->page_count == 0)
1051                 return 0;
1052
1053         if (pg_start < intel_private.base.gtt_stolen_entries) {
1054                 dev_info(&intel_private.pcidev->dev,
1055                          "trying to disable local/stolen memory\n");
1056                 return -EINVAL;
1057         }
1058
1059         if (USE_PCI_DMA_API && INTEL_GTT_GEN > 2)
1060                 intel_agp_unmap_memory(mem);
1061
1062         for (i = pg_start; i < (mem->page_count + pg_start); i++) {
1063                 intel_private.driver->write_entry(intel_private.scratch_page_dma,
1064                                                   i, 0);
1065         }
1066         readl(intel_private.gtt+i-1);
1067
1068         return 0;
1069 }
1070
1071 static void intel_fake_agp_chipset_flush(struct agp_bridge_data *bridge)
1072 {
1073         intel_private.driver->chipset_flush();
1074 }
1075
1076 static struct agp_memory *intel_fake_agp_alloc_by_type(size_t pg_count,
1077                                                        int type)
1078 {
1079         if (type == AGP_PHYS_MEMORY)
1080                 return alloc_agpphysmem_i8xx(pg_count, type);
1081         /* always return NULL for other allocation types for now */
1082         return NULL;
1083 }
1084
1085 static int intel_alloc_chipset_flush_resource(void)
1086 {
1087         int ret;
1088         ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE,
1089                                      PAGE_SIZE, PCIBIOS_MIN_MEM, 0,
1090                                      pcibios_align_resource, intel_private.bridge_dev);
1091
1092         return ret;
1093 }
1094
1095 static void intel_i915_setup_chipset_flush(void)
1096 {
1097         int ret;
1098         u32 temp;
1099
1100         pci_read_config_dword(intel_private.bridge_dev, I915_IFPADDR, &temp);
1101         if (!(temp & 0x1)) {
1102                 intel_alloc_chipset_flush_resource();
1103                 intel_private.resource_valid = 1;
1104                 pci_write_config_dword(intel_private.bridge_dev, I915_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1105         } else {
1106                 temp &= ~1;
1107
1108                 intel_private.resource_valid = 1;
1109                 intel_private.ifp_resource.start = temp;
1110                 intel_private.ifp_resource.end = temp + PAGE_SIZE;
1111                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1112                 /* some BIOSes reserve this area in a pnp some don't */
1113                 if (ret)
1114                         intel_private.resource_valid = 0;
1115         }
1116 }
1117
1118 static void intel_i965_g33_setup_chipset_flush(void)
1119 {
1120         u32 temp_hi, temp_lo;
1121         int ret;
1122
1123         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi);
1124         pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR, &temp_lo);
1125
1126         if (!(temp_lo & 0x1)) {
1127
1128                 intel_alloc_chipset_flush_resource();
1129
1130                 intel_private.resource_valid = 1;
1131                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4,
1132                         upper_32_bits(intel_private.ifp_resource.start));
1133                 pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR, (intel_private.ifp_resource.start & 0xffffffff) | 0x1);
1134         } else {
1135                 u64 l64;
1136
1137                 temp_lo &= ~0x1;
1138                 l64 = ((u64)temp_hi << 32) | temp_lo;
1139
1140                 intel_private.resource_valid = 1;
1141                 intel_private.ifp_resource.start = l64;
1142                 intel_private.ifp_resource.end = l64 + PAGE_SIZE;
1143                 ret = request_resource(&iomem_resource, &intel_private.ifp_resource);
1144                 /* some BIOSes reserve this area in a pnp some don't */
1145                 if (ret)
1146                         intel_private.resource_valid = 0;
1147         }
1148 }
1149
1150 static void intel_i9xx_setup_flush(void)
1151 {
1152         /* return if already configured */
1153         if (intel_private.ifp_resource.start)
1154                 return;
1155
1156         if (INTEL_GTT_GEN == 6)
1157                 return;
1158
1159         /* setup a resource for this object */
1160         intel_private.ifp_resource.name = "Intel Flush Page";
1161         intel_private.ifp_resource.flags = IORESOURCE_MEM;
1162
1163         /* Setup chipset flush for 915 */
1164         if (IS_G33 || INTEL_GTT_GEN >= 4) {
1165                 intel_i965_g33_setup_chipset_flush();
1166         } else {
1167                 intel_i915_setup_chipset_flush();
1168         }
1169
1170         if (intel_private.ifp_resource.start)
1171                 intel_private.i9xx_flush_page = ioremap_nocache(intel_private.ifp_resource.start, PAGE_SIZE);
1172         if (!intel_private.i9xx_flush_page)
1173                 dev_err(&intel_private.pcidev->dev,
1174                         "can't ioremap flush page - no chipset flushing\n");
1175 }
1176
1177 static void i9xx_cleanup(void)
1178 {
1179         if (intel_private.i9xx_flush_page)
1180                 iounmap(intel_private.i9xx_flush_page);
1181         if (intel_private.resource_valid)
1182                 release_resource(&intel_private.ifp_resource);
1183         intel_private.ifp_resource.start = 0;
1184         intel_private.resource_valid = 0;
1185 }
1186
1187 static void i9xx_chipset_flush(void)
1188 {
1189         if (intel_private.i9xx_flush_page)
1190                 writel(1, intel_private.i9xx_flush_page);
1191 }
1192
1193 static void i965_write_entry(dma_addr_t addr, unsigned int entry,
1194                              unsigned int flags)
1195 {
1196         /* Shift high bits down */
1197         addr |= (addr >> 28) & 0xf0;
1198         writel(addr | I810_PTE_VALID, intel_private.gtt + entry);
1199 }
1200
1201 static bool gen6_check_flags(unsigned int flags)
1202 {
1203         return true;
1204 }
1205
1206 static void gen6_write_entry(dma_addr_t addr, unsigned int entry,
1207                              unsigned int flags)
1208 {
1209         unsigned int type_mask = flags & ~AGP_USER_CACHED_MEMORY_GFDT;
1210         unsigned int gfdt = flags & AGP_USER_CACHED_MEMORY_GFDT;
1211         u32 pte_flags;
1212
1213         if (type_mask == AGP_USER_UNCACHED_MEMORY)
1214                 pte_flags = GEN6_PTE_UNCACHED | I810_PTE_VALID;
1215         else if (type_mask == AGP_USER_CACHED_MEMORY_LLC_MLC) {
1216                 pte_flags = GEN6_PTE_LLC | I810_PTE_VALID;
1217                 if (gfdt)
1218                         pte_flags |= GEN6_PTE_GFDT;
1219         } else { /* set 'normal'/'cached' to LLC by default */
1220                 pte_flags = GEN6_PTE_LLC_MLC | I810_PTE_VALID;
1221                 if (gfdt)
1222                         pte_flags |= GEN6_PTE_GFDT;
1223         }
1224
1225         /* gen6 has bit11-4 for physical addr bit39-32 */
1226         addr |= (addr >> 28) & 0xff0;
1227         writel(addr | pte_flags, intel_private.gtt + entry);
1228 }
1229
1230 static void gen6_cleanup(void)
1231 {
1232 }
1233
1234 static int i9xx_setup(void)
1235 {
1236         u32 reg_addr;
1237
1238         pci_read_config_dword(intel_private.pcidev, I915_MMADDR, &reg_addr);
1239
1240         reg_addr &= 0xfff80000;
1241
1242         intel_private.registers = ioremap(reg_addr, 128 * 4096);
1243         if (!intel_private.registers)
1244                 return -ENOMEM;
1245
1246         if (INTEL_GTT_GEN == 3) {
1247                 u32 gtt_addr;
1248
1249                 pci_read_config_dword(intel_private.pcidev,
1250                                       I915_PTEADDR, &gtt_addr);
1251                 intel_private.gtt_bus_addr = gtt_addr;
1252         } else {
1253                 u32 gtt_offset;
1254
1255                 switch (INTEL_GTT_GEN) {
1256                 case 5:
1257                 case 6:
1258                         gtt_offset = MB(2);
1259                         break;
1260                 case 4:
1261                 default:
1262                         gtt_offset =  KB(512);
1263                         break;
1264                 }
1265                 intel_private.gtt_bus_addr = reg_addr + gtt_offset;
1266         }
1267
1268         intel_private.pte_bus_addr =
1269                 readl(intel_private.registers+I810_PGETBL_CTL) & 0xfffff000;
1270
1271         intel_i9xx_setup_flush();
1272
1273         return 0;
1274 }
1275
1276 static const struct agp_bridge_driver intel_810_driver = {
1277         .owner                  = THIS_MODULE,
1278         .aperture_sizes         = intel_i810_sizes,
1279         .size_type              = FIXED_APER_SIZE,
1280         .num_aperture_sizes     = 2,
1281         .needs_scratch_page     = true,
1282         .configure              = intel_i810_configure,
1283         .fetch_size             = intel_i810_fetch_size,
1284         .cleanup                = intel_i810_cleanup,
1285         .mask_memory            = intel_i810_mask_memory,
1286         .masks                  = intel_i810_masks,
1287         .agp_enable             = intel_fake_agp_enable,
1288         .cache_flush            = global_cache_flush,
1289         .create_gatt_table      = agp_generic_create_gatt_table,
1290         .free_gatt_table        = agp_generic_free_gatt_table,
1291         .insert_memory          = intel_i810_insert_entries,
1292         .remove_memory          = intel_i810_remove_entries,
1293         .alloc_by_type          = intel_i810_alloc_by_type,
1294         .free_by_type           = intel_i810_free_by_type,
1295         .agp_alloc_page         = agp_generic_alloc_page,
1296         .agp_alloc_pages        = agp_generic_alloc_pages,
1297         .agp_destroy_page       = agp_generic_destroy_page,
1298         .agp_destroy_pages      = agp_generic_destroy_pages,
1299         .agp_type_to_mask_type  = agp_generic_type_to_mask_type,
1300 };
1301
1302 static const struct agp_bridge_driver intel_fake_agp_driver = {
1303         .owner                  = THIS_MODULE,
1304         .size_type              = FIXED_APER_SIZE,
1305         .aperture_sizes         = intel_fake_agp_sizes,
1306         .num_aperture_sizes     = ARRAY_SIZE(intel_fake_agp_sizes),
1307         .configure              = intel_fake_agp_configure,
1308         .fetch_size             = intel_fake_agp_fetch_size,
1309         .cleanup                = intel_gtt_cleanup,
1310         .agp_enable             = intel_fake_agp_enable,
1311         .cache_flush            = global_cache_flush,
1312         .create_gatt_table      = intel_fake_agp_create_gatt_table,
1313         .free_gatt_table        = intel_fake_agp_free_gatt_table,
1314         .insert_memory          = intel_fake_agp_insert_entries,
1315         .remove_memory          = intel_fake_agp_remove_entries,
1316         .alloc_by_type          = intel_fake_agp_alloc_by_type,
1317         .free_by_type           = intel_i810_free_by_type,
1318         .agp_alloc_page         = agp_generic_alloc_page,
1319         .agp_alloc_pages        = agp_generic_alloc_pages,
1320         .agp_destroy_page       = agp_generic_destroy_page,
1321         .agp_destroy_pages      = agp_generic_destroy_pages,
1322         .chipset_flush          = intel_fake_agp_chipset_flush,
1323 };
1324
1325 static const struct intel_gtt_driver i81x_gtt_driver = {
1326         .gen = 1,
1327         .dma_mask_size = 32,
1328 };
1329 static const struct intel_gtt_driver i8xx_gtt_driver = {
1330         .gen = 2,
1331         .setup = i830_setup,
1332         .cleanup = i830_cleanup,
1333         .write_entry = i830_write_entry,
1334         .dma_mask_size = 32,
1335         .check_flags = i830_check_flags,
1336         .chipset_flush = i830_chipset_flush,
1337 };
1338 static const struct intel_gtt_driver i915_gtt_driver = {
1339         .gen = 3,
1340         .setup = i9xx_setup,
1341         .cleanup = i9xx_cleanup,
1342         /* i945 is the last gpu to need phys mem (for overlay and cursors). */
1343         .write_entry = i830_write_entry,
1344         .dma_mask_size = 32,
1345         .check_flags = i830_check_flags,
1346         .chipset_flush = i9xx_chipset_flush,
1347 };
1348 static const struct intel_gtt_driver g33_gtt_driver = {
1349         .gen = 3,
1350         .is_g33 = 1,
1351         .setup = i9xx_setup,
1352         .cleanup = i9xx_cleanup,
1353         .write_entry = i965_write_entry,
1354         .dma_mask_size = 36,
1355         .check_flags = i830_check_flags,
1356         .chipset_flush = i9xx_chipset_flush,
1357 };
1358 static const struct intel_gtt_driver pineview_gtt_driver = {
1359         .gen = 3,
1360         .is_pineview = 1, .is_g33 = 1,
1361         .setup = i9xx_setup,
1362         .cleanup = i9xx_cleanup,
1363         .write_entry = i965_write_entry,
1364         .dma_mask_size = 36,
1365         .check_flags = i830_check_flags,
1366         .chipset_flush = i9xx_chipset_flush,
1367 };
1368 static const struct intel_gtt_driver i965_gtt_driver = {
1369         .gen = 4,
1370         .setup = i9xx_setup,
1371         .cleanup = i9xx_cleanup,
1372         .write_entry = i965_write_entry,
1373         .dma_mask_size = 36,
1374         .check_flags = i830_check_flags,
1375         .chipset_flush = i9xx_chipset_flush,
1376 };
1377 static const struct intel_gtt_driver g4x_gtt_driver = {
1378         .gen = 5,
1379         .setup = i9xx_setup,
1380         .cleanup = i9xx_cleanup,
1381         .write_entry = i965_write_entry,
1382         .dma_mask_size = 36,
1383         .check_flags = i830_check_flags,
1384         .chipset_flush = i9xx_chipset_flush,
1385 };
1386 static const struct intel_gtt_driver ironlake_gtt_driver = {
1387         .gen = 5,
1388         .is_ironlake = 1,
1389         .setup = i9xx_setup,
1390         .cleanup = i9xx_cleanup,
1391         .write_entry = i965_write_entry,
1392         .dma_mask_size = 36,
1393         .check_flags = i830_check_flags,
1394         .chipset_flush = i9xx_chipset_flush,
1395 };
1396 static const struct intel_gtt_driver sandybridge_gtt_driver = {
1397         .gen = 6,
1398         .setup = i9xx_setup,
1399         .cleanup = gen6_cleanup,
1400         .write_entry = gen6_write_entry,
1401         .dma_mask_size = 40,
1402         .check_flags = gen6_check_flags,
1403         .chipset_flush = i9xx_chipset_flush,
1404 };
1405
1406 /* Table to describe Intel GMCH and AGP/PCIE GART drivers.  At least one of
1407  * driver and gmch_driver must be non-null, and find_gmch will determine
1408  * which one should be used if a gmch_chip_id is present.
1409  */
1410 static const struct intel_gtt_driver_description {
1411         unsigned int gmch_chip_id;
1412         char *name;
1413         const struct agp_bridge_driver *gmch_driver;
1414         const struct intel_gtt_driver *gtt_driver;
1415 } intel_gtt_chipsets[] = {
1416         { PCI_DEVICE_ID_INTEL_82810_IG1, "i810", &intel_810_driver,
1417                 &i81x_gtt_driver},
1418         { PCI_DEVICE_ID_INTEL_82810_IG3, "i810", &intel_810_driver,
1419                 &i81x_gtt_driver},
1420         { PCI_DEVICE_ID_INTEL_82810E_IG, "i810", &intel_810_driver,
1421                 &i81x_gtt_driver},
1422         { PCI_DEVICE_ID_INTEL_82815_CGC, "i815", &intel_810_driver,
1423                 &i81x_gtt_driver},
1424         { PCI_DEVICE_ID_INTEL_82830_CGC, "830M",
1425                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1426         { PCI_DEVICE_ID_INTEL_82845G_IG, "830M",
1427                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1428         { PCI_DEVICE_ID_INTEL_82854_IG, "854",
1429                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1430         { PCI_DEVICE_ID_INTEL_82855GM_IG, "855GM",
1431                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1432         { PCI_DEVICE_ID_INTEL_82865_IG, "865",
1433                 &intel_fake_agp_driver, &i8xx_gtt_driver},
1434         { PCI_DEVICE_ID_INTEL_E7221_IG, "E7221 (i915)",
1435                 &intel_fake_agp_driver, &i915_gtt_driver },
1436         { PCI_DEVICE_ID_INTEL_82915G_IG, "915G",
1437                 &intel_fake_agp_driver, &i915_gtt_driver },
1438         { PCI_DEVICE_ID_INTEL_82915GM_IG, "915GM",
1439                 &intel_fake_agp_driver, &i915_gtt_driver },
1440         { PCI_DEVICE_ID_INTEL_82945G_IG, "945G",
1441                 &intel_fake_agp_driver, &i915_gtt_driver },
1442         { PCI_DEVICE_ID_INTEL_82945GM_IG, "945GM",
1443                 &intel_fake_agp_driver, &i915_gtt_driver },
1444         { PCI_DEVICE_ID_INTEL_82945GME_IG, "945GME",
1445                 &intel_fake_agp_driver, &i915_gtt_driver },
1446         { PCI_DEVICE_ID_INTEL_82946GZ_IG, "946GZ",
1447                 &intel_fake_agp_driver, &i965_gtt_driver },
1448         { PCI_DEVICE_ID_INTEL_82G35_IG, "G35",
1449                 &intel_fake_agp_driver, &i965_gtt_driver },
1450         { PCI_DEVICE_ID_INTEL_82965Q_IG, "965Q",
1451                 &intel_fake_agp_driver, &i965_gtt_driver },
1452         { PCI_DEVICE_ID_INTEL_82965G_IG, "965G",
1453                 &intel_fake_agp_driver, &i965_gtt_driver },
1454         { PCI_DEVICE_ID_INTEL_82965GM_IG, "965GM",
1455                 &intel_fake_agp_driver, &i965_gtt_driver },
1456         { PCI_DEVICE_ID_INTEL_82965GME_IG, "965GME/GLE",
1457                 &intel_fake_agp_driver, &i965_gtt_driver },
1458         { PCI_DEVICE_ID_INTEL_G33_IG, "G33",
1459                 &intel_fake_agp_driver, &g33_gtt_driver },
1460         { PCI_DEVICE_ID_INTEL_Q35_IG, "Q35",
1461                 &intel_fake_agp_driver, &g33_gtt_driver },
1462         { PCI_DEVICE_ID_INTEL_Q33_IG, "Q33",
1463                 &intel_fake_agp_driver, &g33_gtt_driver },
1464         { PCI_DEVICE_ID_INTEL_PINEVIEW_M_IG, "GMA3150",
1465                 &intel_fake_agp_driver, &pineview_gtt_driver },
1466         { PCI_DEVICE_ID_INTEL_PINEVIEW_IG, "GMA3150",
1467                 &intel_fake_agp_driver, &pineview_gtt_driver },
1468         { PCI_DEVICE_ID_INTEL_GM45_IG, "GM45",
1469                 &intel_fake_agp_driver, &g4x_gtt_driver },
1470         { PCI_DEVICE_ID_INTEL_EAGLELAKE_IG, "Eaglelake",
1471                 &intel_fake_agp_driver, &g4x_gtt_driver },
1472         { PCI_DEVICE_ID_INTEL_Q45_IG, "Q45/Q43",
1473                 &intel_fake_agp_driver, &g4x_gtt_driver },
1474         { PCI_DEVICE_ID_INTEL_G45_IG, "G45/G43",
1475                 &intel_fake_agp_driver, &g4x_gtt_driver },
1476         { PCI_DEVICE_ID_INTEL_B43_IG, "B43",
1477                 &intel_fake_agp_driver, &g4x_gtt_driver },
1478         { PCI_DEVICE_ID_INTEL_B43_1_IG, "B43",
1479                 &intel_fake_agp_driver, &g4x_gtt_driver },
1480         { PCI_DEVICE_ID_INTEL_G41_IG, "G41",
1481                 &intel_fake_agp_driver, &g4x_gtt_driver },
1482         { PCI_DEVICE_ID_INTEL_IRONLAKE_D_IG,
1483             "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver },
1484         { PCI_DEVICE_ID_INTEL_IRONLAKE_M_IG,
1485             "HD Graphics", &intel_fake_agp_driver, &ironlake_gtt_driver },
1486         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT1_IG,
1487             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1488         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_IG,
1489             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1490         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_GT2_PLUS_IG,
1491             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1492         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT1_IG,
1493             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1494         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_IG,
1495             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1496         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_M_GT2_PLUS_IG,
1497             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1498         { PCI_DEVICE_ID_INTEL_SANDYBRIDGE_S_IG,
1499             "Sandybridge", &intel_fake_agp_driver, &sandybridge_gtt_driver },
1500         { 0, NULL, NULL }
1501 };
1502
1503 static int find_gmch(u16 device)
1504 {
1505         struct pci_dev *gmch_device;
1506
1507         gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
1508         if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) {
1509                 gmch_device = pci_get_device(PCI_VENDOR_ID_INTEL,
1510                                              device, gmch_device);
1511         }
1512
1513         if (!gmch_device)
1514                 return 0;
1515
1516         intel_private.pcidev = gmch_device;
1517         return 1;
1518 }
1519
1520 int intel_gmch_probe(struct pci_dev *pdev,
1521                                       struct agp_bridge_data *bridge)
1522 {
1523         int i, mask;
1524         bridge->driver = NULL;
1525
1526         for (i = 0; intel_gtt_chipsets[i].name != NULL; i++) {
1527                 if (find_gmch(intel_gtt_chipsets[i].gmch_chip_id)) {
1528                         bridge->driver =
1529                                 intel_gtt_chipsets[i].gmch_driver;
1530                         intel_private.driver =
1531                                 intel_gtt_chipsets[i].gtt_driver;
1532                         break;
1533                 }
1534         }
1535
1536         if (!bridge->driver)
1537                 return 0;
1538
1539         bridge->dev_private_data = &intel_private;
1540         bridge->dev = pdev;
1541
1542         intel_private.bridge_dev = pci_dev_get(pdev);
1543
1544         dev_info(&pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name);
1545
1546         mask = intel_private.driver->dma_mask_size;
1547         if (pci_set_dma_mask(intel_private.pcidev, DMA_BIT_MASK(mask)))
1548                 dev_err(&intel_private.pcidev->dev,
1549                         "set gfx device dma mask %d-bit failed!\n", mask);
1550         else
1551                 pci_set_consistent_dma_mask(intel_private.pcidev,
1552                                             DMA_BIT_MASK(mask));
1553
1554         if (bridge->driver == &intel_810_driver)
1555                 return 1;
1556
1557         if (intel_gtt_init() != 0)
1558                 return 0;
1559
1560         return 1;
1561 }
1562 EXPORT_SYMBOL(intel_gmch_probe);
1563
1564 struct intel_gtt *intel_gtt_get(void)
1565 {
1566         return &intel_private.base;
1567 }
1568 EXPORT_SYMBOL(intel_gtt_get);
1569
1570 void intel_gmch_remove(struct pci_dev *pdev)
1571 {
1572         if (intel_private.pcidev)
1573                 pci_dev_put(intel_private.pcidev);
1574         if (intel_private.bridge_dev)
1575                 pci_dev_put(intel_private.bridge_dev);
1576 }
1577 EXPORT_SYMBOL(intel_gmch_remove);
1578
1579 MODULE_AUTHOR("Dave Jones <davej@redhat.com>");
1580 MODULE_LICENSE("GPL and additional rights");