drivers/edac/i7core_edac.c

   1 /* Intel 7 core  Memory Controller kernel module (Nehalem)
   2  *
   3  * This file may be distributed under the terms of the
   4  * GNU General Public License version 2 only.
   5  *
   6  * Copyright (c) 2009 by:
   7  *       Mauro Carvalho Chehab <mchehab@redhat.com>
   8  *
   9  * Red Hat Inc. http://www.redhat.com
  10  *
  11  * Forked and adapted from the i5400_edac driver
  12  *
  13  * Based on the following public Intel datasheets:
  14  * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
  15  * Datasheet, Volume 2:
  16  *      http://download.intel.com/design/processor/datashts/320835.pdf
  17  * Intel Xeon Processor 5500 Series Datasheet Volume 2
  18  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
  19  * also available at:
  20  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  21  */
  22
  23 #include <linux/module.h>
  24 #include <linux/init.h>
  25 #include <linux/pci.h>
  26 #include <linux/pci_ids.h>
  27 #include <linux/slab.h>
  28 #include <linux/edac.h>
  29 #include <linux/mmzone.h>
  30 #include <linux/edac_mce.h>
  31 #include <linux/spinlock.h>
  32 #include <linux/smp.h>
  33 #include <asm/processor.h>
  34
  35 #include "edac_core.h"
  36
  37 /*
  38  * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
  39  * registers start at bus 255, and are not reported by BIOS.
  40  * We currently find devices with only 2 sockets. In order to support more QPI
  41  * Quick Path Interconnect, just increment this number.
  42  */
  43 #define MAX_SOCKET_BUSES        2
  44
  45
  46 /*
  47  * Alter this version for the module when modifications are made
  48  */
  49 #define I7CORE_REVISION    " Ver: 1.0.0 " __DATE__
  50 #define EDAC_MOD_STR      "i7core_edac"
  51
  52 /*
  53  * Debug macros
  54  */
  55 #define i7core_printk(level, fmt, arg...)                       \
  56         edac_printk(level, "i7core", fmt, ##arg)
  57
  58 #define i7core_mc_printk(mci, level, fmt, arg...)               \
  59         edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
  60
  61 /*
  62  * i7core Memory Controller Registers
  63  */
  64
  65         /* OFFSETS for Device 0 Function 0 */
  66
  67 #define MC_CFG_CONTROL  0x90
  68
  69         /* OFFSETS for Device 3 Function 0 */
  70
  71 #define MC_CONTROL      0x48
  72 #define MC_STATUS       0x4c
  73 #define MC_MAX_DOD      0x64
  74
  75 /*
  76  * OFFSETS for Device 3 Function 4, as inicated on Xeon 5500 datasheet:
  77  * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  78  */
  79
  80 #define MC_TEST_ERR_RCV1        0x60
  81   #define DIMM2_COR_ERR(r)                      ((r) & 0x7fff)
  82
  83 #define MC_TEST_ERR_RCV0        0x64
  84   #define DIMM1_COR_ERR(r)                      (((r) >> 16) & 0x7fff)
  85   #define DIMM0_COR_ERR(r)                      ((r) & 0x7fff)
  86
  87 /* OFFSETS for Device 3 Function 2, as inicated on Xeon 5500 datasheet */
  88 #define MC_COR_ECC_CNT_0        0x80
  89 #define MC_COR_ECC_CNT_1        0x84
  90 #define MC_COR_ECC_CNT_2        0x88
  91 #define MC_COR_ECC_CNT_3        0x8c
  92 #define MC_COR_ECC_CNT_4        0x90
  93 #define MC_COR_ECC_CNT_5        0x94
  94
  95 #define DIMM_TOP_COR_ERR(r)                     (((r) >> 16) & 0x7fff)
  96 #define DIMM_BOT_COR_ERR(r)                     ((r) & 0x7fff)
  97
  98
  99         /* OFFSETS for Devices 4,5 and 6 Function 0 */
 100
 101 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
 102   #define THREE_DIMMS_PRESENT           (1 << 24)
 103   #define SINGLE_QUAD_RANK_PRESENT      (1 << 23)
 104   #define QUAD_RANK_PRESENT             (1 << 22)
 105   #define REGISTERED_DIMM               (1 << 15)
 106
 107 #define MC_CHANNEL_MAPPER       0x60
 108   #define RDLCH(r, ch)          ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
 109   #define WRLCH(r, ch)          ((((r) >> (ch * 6)) & 0x07) - 1)
 110
 111 #define MC_CHANNEL_RANK_PRESENT 0x7c
 112   #define RANK_PRESENT_MASK             0xffff
 113
 114 #define MC_CHANNEL_ADDR_MATCH   0xf0
 115 #define MC_CHANNEL_ERROR_MASK   0xf8
 116 #define MC_CHANNEL_ERROR_INJECT 0xfc
 117   #define INJECT_ADDR_PARITY    0x10
 118   #define INJECT_ECC            0x08
 119   #define MASK_CACHELINE        0x06
 120   #define MASK_FULL_CACHELINE   0x06
 121   #define MASK_MSB32_CACHELINE  0x04
 122   #define MASK_LSB32_CACHELINE  0x02
 123   #define NO_MASK_CACHELINE     0x00
 124   #define REPEAT_EN             0x01
 125
 126         /* OFFSETS for Devices 4,5 and 6 Function 1 */
 127
 128 #define MC_DOD_CH_DIMM0         0x48
 129 #define MC_DOD_CH_DIMM1         0x4c
 130 #define MC_DOD_CH_DIMM2         0x50
 131   #define RANKOFFSET_MASK       ((1 << 12) | (1 << 11) | (1 << 10))
 132   #define RANKOFFSET(x)         ((x & RANKOFFSET_MASK) >> 10)
 133   #define DIMM_PRESENT_MASK     (1 << 9)
 134   #define DIMM_PRESENT(x)       (((x) & DIMM_PRESENT_MASK) >> 9)
 135   #define MC_DOD_NUMBANK_MASK           ((1 << 8) | (1 << 7))
 136   #define MC_DOD_NUMBANK(x)             (((x) & MC_DOD_NUMBANK_MASK) >> 7)
 137   #define MC_DOD_NUMRANK_MASK           ((1 << 6) | (1 << 5))
 138   #define MC_DOD_NUMRANK(x)             (((x) & MC_DOD_NUMRANK_MASK) >> 5)
 139   #define MC_DOD_NUMROW_MASK            ((1 << 4) | (1 << 3) | (1 << 2))
 140   #define MC_DOD_NUMROW(x)              (((x) & MC_DOD_NUMROW_MASK) >> 2)
 141   #define MC_DOD_NUMCOL_MASK            3
 142   #define MC_DOD_NUMCOL(x)              ((x) & MC_DOD_NUMCOL_MASK)
 143
 144 #define MC_RANK_PRESENT         0x7c
 145
 146 #define MC_SAG_CH_0     0x80
 147 #define MC_SAG_CH_1     0x84
 148 #define MC_SAG_CH_2     0x88
 149 #define MC_SAG_CH_3     0x8c
 150 #define MC_SAG_CH_4     0x90
 151 #define MC_SAG_CH_5     0x94
 152 #define MC_SAG_CH_6     0x98
 153 #define MC_SAG_CH_7     0x9c
 154
 155 #define MC_RIR_LIMIT_CH_0       0x40
 156 #define MC_RIR_LIMIT_CH_1       0x44
 157 #define MC_RIR_LIMIT_CH_2       0x48
 158 #define MC_RIR_LIMIT_CH_3       0x4C
 159 #define MC_RIR_LIMIT_CH_4       0x50
 160 #define MC_RIR_LIMIT_CH_5       0x54
 161 #define MC_RIR_LIMIT_CH_6       0x58
 162 #define MC_RIR_LIMIT_CH_7       0x5C
 163 #define MC_RIR_LIMIT_MASK       ((1 << 10) - 1)
 164
 165 #define MC_RIR_WAY_CH           0x80
 166   #define MC_RIR_WAY_OFFSET_MASK        (((1 << 14) - 1) & ~0x7)
 167   #define MC_RIR_WAY_RANK_MASK          0x7
 168
 169 /*
 170  * i7core structs
 171  */
 172
 173 #define NUM_CHANS 3
 174 #define MAX_DIMMS 3             /* Max DIMMS per channel */
 175 #define MAX_MCR_FUNC  4
 176 #define MAX_CHAN_FUNC 3
 177
 178 struct i7core_info {
 179         u32     mc_control;
 180         u32     mc_status;
 181         u32     max_dod;
 182         u32     ch_map;
 183 };
 184
 185
 186 struct i7core_inject {
 187         int     enable;
 188
 189         u32     section;
 190         u32     type;
 191         u32     eccmask;
 192
 193         /* Error address mask */
 194         int channel, dimm, rank, bank, page, col;
 195 };
 196
 197 struct i7core_channel {
 198         u32             ranks;
 199         u32             dimms;
 200 };
 201
 202 struct pci_id_descr {
 203         int                     dev;
 204         int                     func;
 205         int                     dev_id;
 206 };
 207
 208 struct i7core_dev {
 209         struct list_head        list;
 210         u8                      socket;
 211         struct pci_dev          **pdev;
 212         struct mem_ctl_info     *mci;
 213 };
 214
 215 struct i7core_pvt {
 216         struct pci_dev  *pci_noncore;
 217         struct pci_dev  *pci_mcr[MAX_MCR_FUNC + 1];
 218         struct pci_dev  *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
 219
 220         struct i7core_dev *i7core_dev;
 221
 222         struct i7core_info      info;
 223         struct i7core_inject    inject;
 224         struct i7core_channel   channel[NUM_CHANS];
 225
 226         int             channels; /* Number of active channels */
 227
 228         int             ce_count_available;
 229         int             csrow_map[NUM_CHANS][MAX_DIMMS];
 230
 231                         /* ECC corrected errors counts per udimm */
 232         unsigned long   udimm_ce_count[MAX_DIMMS];
 233         int             udimm_last_ce_count[MAX_DIMMS];
 234                         /* ECC corrected errors counts per rdimm */
 235         unsigned long   rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
 236         int             rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
 237
 238         unsigned int    is_registered;
 239
 240         /* mcelog glue */
 241         struct edac_mce         edac_mce;
 242         struct mce              mce_entry[MCE_LOG_LEN];
 243         unsigned                mce_count;
 244         spinlock_t              mce_lock;
 245 };
 246
 247 /* Static vars */
 248 static LIST_HEAD(i7core_edac_list);
 249 static DEFINE_MUTEX(i7core_edac_lock);
 250 static u8 max_num_sockets;
 251
 252 #define PCI_DESCR(device, function, device_id)  \
 253         .dev = (device),                        \
 254         .func = (function),                     \
 255         .dev_id = (device_id)
 256
 257 struct pci_id_descr pci_dev_descr[] = {
 258                 /* Memory controller */
 259         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
 260         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
 261         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS)  }, /* if RDIMM */
 262         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
 263
 264                 /* Channel 0 */
 265         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
 266         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
 267         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
 268         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
 269
 270                 /* Channel 1 */
 271         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
 272         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
 273         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
 274         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
 275
 276                 /* Channel 2 */
 277         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
 278         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
 279         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
 280         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
 281
 282                 /* Generic Non-core registers */
 283         /*
 284          * This is the PCI device on i7core and on Xeon 35xx (8086:2c41)
 285          * On Xeon 55xx, however, it has a different id (8086:2c40). So,
 286          * the probing code needs to test for the other address in case of
 287          * failure of this one
 288          */
 289         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NOCORE)  },
 290
 291 };
 292 #define N_DEVS ARRAY_SIZE(pci_dev_descr)
 293
 294 /*
 295  *      pci_device_id   table for which devices we are looking for
 296  */
 297 static const struct pci_device_id i7core_pci_tbl[] __devinitdata = {
 298         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
 299         {0,}                    /* 0 terminated list. */
 300 };
 301
 302 static struct edac_pci_ctl_info *i7core_pci;
 303
 304 /****************************************************************************
 305                         Anciliary status routines
 306  ****************************************************************************/
 307
 308         /* MC_CONTROL bits */
 309 #define CH_ACTIVE(pvt, ch)      ((pvt)->info.mc_control & (1 << (8 + ch)))
 310 #define ECCx8(pvt)              ((pvt)->info.mc_control & (1 << 1))
 311
 312         /* MC_STATUS bits */
 313 #define ECC_ENABLED(pvt)        ((pvt)->info.mc_status & (1 << 4))
 314 #define CH_DISABLED(pvt, ch)    ((pvt)->info.mc_status & (1 << ch))
 315
 316         /* MC_MAX_DOD read functions */
 317 static inline int numdimms(u32 dimms)
 318 {
 319         return (dimms & 0x3) + 1;
 320 }
 321
 322 static inline int numrank(u32 rank)
 323 {
 324         static int ranks[4] = { 1, 2, 4, -EINVAL };
 325
 326         return ranks[rank & 0x3];
 327 }
 328
 329 static inline int numbank(u32 bank)
 330 {
 331         static int banks[4] = { 4, 8, 16, -EINVAL };
 332
 333         return banks[bank & 0x3];
 334 }
 335
 336 static inline int numrow(u32 row)
 337 {
 338         static int rows[8] = {
 339                 1 << 12, 1 << 13, 1 << 14, 1 << 15,
 340                 1 << 16, -EINVAL, -EINVAL, -EINVAL,
 341         };
 342
 343         return rows[row & 0x7];
 344 }
 345
 346 static inline int numcol(u32 col)
 347 {
 348         static int cols[8] = {
 349                 1 << 10, 1 << 11, 1 << 12, -EINVAL,
 350         };
 351         return cols[col & 0x3];
 352 }
 353
 354 static struct i7core_dev *get_i7core_dev(u8 socket)
 355 {
 356         struct i7core_dev *i7core_dev;
 357
 358         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
 359                 if (i7core_dev->socket == socket)
 360                         return i7core_dev;
 361         }
 362
 363         return NULL;
 364 }
 365
 366 /****************************************************************************
 367                         Memory check routines
 368  ****************************************************************************/
 369 static struct pci_dev *get_pdev_slot_func(u8 socket, unsigned slot,
 370                                           unsigned func)
 371 {
 372         struct i7core_dev *i7core_dev = get_i7core_dev(socket);
 373         int i;
 374
 375         if (!i7core_dev)
 376                 return NULL;
 377
 378         for (i = 0; i < N_DEVS; i++) {
 379                 if (!i7core_dev->pdev[i])
 380                         continue;
 381
 382                 if (PCI_SLOT(i7core_dev->pdev[i]->devfn) == slot &&
 383                     PCI_FUNC(i7core_dev->pdev[i]->devfn) == func) {
 384                         return i7core_dev->pdev[i];
 385                 }
 386         }
 387
 388         return NULL;
 389 }
 390
 391 /**
 392  * i7core_get_active_channels() - gets the number of channels and csrows
 393  * @socket:     Quick Path Interconnect socket
 394  * @channels:   Number of channels that will be returned
 395  * @csrows:     Number of csrows found
 396  *
 397  * Since EDAC core needs to know in advance the number of available channels
 398  * and csrows, in order to allocate memory for csrows/channels, it is needed
 399  * to run two similar steps. At the first step, implemented on this function,
 400  * it checks the number of csrows/channels present at one socket.
 401  * this is used in order to properly allocate the size of mci components.
 402  *
 403  * It should be noticed that none of the current available datasheets explain
 404  * or even mention how csrows are seen by the memory controller. So, we need
 405  * to add a fake description for csrows.
 406  * So, this driver is attributing one DIMM memory for one csrow.
 407  */
 408 static int i7core_get_active_channels(u8 socket, unsigned *channels,
 409                                       unsigned *csrows)
 410 {
 411         struct pci_dev *pdev = NULL;
 412         int i, j;
 413         u32 status, control;
 414
 415         *channels = 0;
 416         *csrows = 0;
 417
 418         pdev = get_pdev_slot_func(socket, 3, 0);
 419         if (!pdev) {
 420                 i7core_printk(KERN_ERR, "Couldn't find socket %d fn 3.0!!!\n",
 421                               socket);
 422                 return -ENODEV;
 423         }
 424
 425         /* Device 3 function 0 reads */
 426         pci_read_config_dword(pdev, MC_STATUS, &status);
 427         pci_read_config_dword(pdev, MC_CONTROL, &control);
 428
 429         for (i = 0; i < NUM_CHANS; i++) {
 430                 u32 dimm_dod[3];
 431                 /* Check if the channel is active */
 432                 if (!(control & (1 << (8 + i))))
 433                         continue;
 434
 435                 /* Check if the channel is disabled */
 436                 if (status & (1 << i))
 437                         continue;
 438
 439                 pdev = get_pdev_slot_func(socket, i + 4, 1);
 440                 if (!pdev) {
 441                         i7core_printk(KERN_ERR, "Couldn't find socket %d "
 442                                                 "fn %d.%d!!!\n",
 443                                                 socket, i + 4, 1);
 444                         return -ENODEV;
 445                 }
 446                 /* Devices 4-6 function 1 */
 447                 pci_read_config_dword(pdev,
 448                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 449                 pci_read_config_dword(pdev,
 450                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 451                 pci_read_config_dword(pdev,
 452                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 453
 454                 (*channels)++;
 455
 456                 for (j = 0; j < 3; j++) {
 457                         if (!DIMM_PRESENT(dimm_dod[j]))
 458                                 continue;
 459                         (*csrows)++;
 460                 }
 461         }
 462
 463         debugf0("Number of active channels on socket %d: %d\n",
 464                 socket, *channels);
 465
 466         return 0;
 467 }
 468
 469 static int get_dimm_config(struct mem_ctl_info *mci, int *csrow)
 470 {
 471         struct i7core_pvt *pvt = mci->pvt_info;
 472         struct csrow_info *csr;
 473         struct pci_dev *pdev;
 474         int i, j;
 475         u8 socket = pvt->i7core_dev->socket;
 476         unsigned long last_page = 0;
 477         enum edac_type mode;
 478         enum mem_type mtype;
 479
 480         /* Get data from the MC register, function 0 */
 481         pdev = pvt->pci_mcr[0];
 482         if (!pdev)
 483                 return -ENODEV;
 484
 485         /* Device 3 function 0 reads */
 486         pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
 487         pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
 488         pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
 489         pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
 490
 491         debugf0("QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
 492                 socket, pvt->info.mc_control, pvt->info.mc_status,
 493                 pvt->info.max_dod, pvt->info.ch_map);
 494
 495         if (ECC_ENABLED(pvt)) {
 496                 debugf0("ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
 497                 if (ECCx8(pvt))
 498                         mode = EDAC_S8ECD8ED;
 499                 else
 500                         mode = EDAC_S4ECD4ED;
 501         } else {
 502                 debugf0("ECC disabled\n");
 503                 mode = EDAC_NONE;
 504         }
 505
 506         /* FIXME: need to handle the error codes */
 507         debugf0("DOD Max limits: DIMMS: %d, %d-ranked, %d-banked "
 508                 "x%x x 0x%x\n",
 509                 numdimms(pvt->info.max_dod),
 510                 numrank(pvt->info.max_dod >> 2),
 511                 numbank(pvt->info.max_dod >> 4),
 512                 numrow(pvt->info.max_dod >> 6),
 513                 numcol(pvt->info.max_dod >> 9));
 514
 515         for (i = 0; i < NUM_CHANS; i++) {
 516                 u32 data, dimm_dod[3], value[8];
 517
 518                 if (!CH_ACTIVE(pvt, i)) {
 519                         debugf0("Channel %i is not active\n", i);
 520                         continue;
 521                 }
 522                 if (CH_DISABLED(pvt, i)) {
 523                         debugf0("Channel %i is disabled\n", i);
 524                         continue;
 525                 }
 526
 527                 /* Devices 4-6 function 0 */
 528                 pci_read_config_dword(pvt->pci_ch[i][0],
 529                                 MC_CHANNEL_DIMM_INIT_PARAMS, &data);
 530
 531                 pvt->channel[i].ranks = (data & QUAD_RANK_PRESENT) ?
 532                                                 4 : 2;
 533
 534                 if (data & REGISTERED_DIMM)
 535                         mtype = MEM_RDDR3;
 536                 else
 537                         mtype = MEM_DDR3;
 538 #if 0
 539                 if (data & THREE_DIMMS_PRESENT)
 540                         pvt->channel[i].dimms = 3;
 541                 else if (data & SINGLE_QUAD_RANK_PRESENT)
 542                         pvt->channel[i].dimms = 1;
 543                 else
 544                         pvt->channel[i].dimms = 2;
 545 #endif
 546
 547                 /* Devices 4-6 function 1 */
 548                 pci_read_config_dword(pvt->pci_ch[i][1],
 549                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 550                 pci_read_config_dword(pvt->pci_ch[i][1],
 551                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 552                 pci_read_config_dword(pvt->pci_ch[i][1],
 553                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 554
 555                 debugf0("Ch%d phy rd%d, wr%d (0x%08x): "
 556                         "%d ranks, %cDIMMs\n",
 557                         i,
 558                         RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
 559                         data,
 560                         pvt->channel[i].ranks,
 561                         (data & REGISTERED_DIMM) ? 'R' : 'U');
 562
 563                 for (j = 0; j < 3; j++) {
 564                         u32 banks, ranks, rows, cols;
 565                         u32 size, npages;
 566
 567                         if (!DIMM_PRESENT(dimm_dod[j]))
 568                                 continue;
 569
 570                         banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
 571                         ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
 572                         rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
 573                         cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
 574
 575                         /* DDR3 has 8 I/O banks */
 576                         size = (rows * cols * banks * ranks) >> (20 - 3);
 577
 578                         pvt->channel[i].dimms++;
 579
 580                         debugf0("\tdimm %d %d Mb offset: %x, "
 581                                 "bank: %d, rank: %d, row: %#x, col: %#x\n",
 582                                 j, size,
 583                                 RANKOFFSET(dimm_dod[j]),
 584                                 banks, ranks, rows, cols);
 585
 586 #if PAGE_SHIFT > 20
 587                         npages = size >> (PAGE_SHIFT - 20);
 588 #else
 589                         npages = size << (20 - PAGE_SHIFT);
 590 #endif
 591
 592                         csr = &mci->csrows[*csrow];
 593                         csr->first_page = last_page + 1;
 594                         last_page += npages;
 595                         csr->last_page = last_page;
 596                         csr->nr_pages = npages;
 597
 598                         csr->page_mask = 0;
 599                         csr->grain = 8;
 600                         csr->csrow_idx = *csrow;
 601                         csr->nr_channels = 1;
 602
 603                         csr->channels[0].chan_idx = i;
 604                         csr->channels[0].ce_count = 0;
 605
 606                         pvt->csrow_map[i][j] = *csrow;
 607
 608                         switch (banks) {
 609                         case 4:
 610                                 csr->dtype = DEV_X4;
 611                                 break;
 612                         case 8:
 613                                 csr->dtype = DEV_X8;
 614                                 break;
 615                         case 16:
 616                                 csr->dtype = DEV_X16;
 617                                 break;
 618                         default:
 619                                 csr->dtype = DEV_UNKNOWN;
 620                         }
 621
 622                         csr->edac_mode = mode;
 623                         csr->mtype = mtype;
 624
 625                         (*csrow)++;
 626                 }
 627
 628                 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
 629                 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
 630                 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
 631                 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
 632                 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
 633                 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
 634                 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
 635                 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
 636                 debugf1("\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
 637                 for (j = 0; j < 8; j++)
 638                         debugf1("\t\t%#x\t%#x\t%#x\n",
 639                                 (value[j] >> 27) & 0x1,
 640                                 (value[j] >> 24) & 0x7,
 641                                 (value[j] && ((1 << 24) - 1)));
 642         }
 643
 644         return 0;
 645 }
 646
 647 /****************************************************************************
 648                         Error insertion routines
 649  ****************************************************************************/
 650
 651 /* The i7core has independent error injection features per channel.
 652    However, to have a simpler code, we don't allow enabling error injection
 653    on more than one channel.
 654    Also, since a change at an inject parameter will be applied only at enable,
 655    we're disabling error injection on all write calls to the sysfs nodes that
 656    controls the error code injection.
 657  */
 658 static int disable_inject(struct mem_ctl_info *mci)
 659 {
 660         struct i7core_pvt *pvt = mci->pvt_info;
 661
 662         pvt->inject.enable = 0;
 663
 664         if (!pvt->pci_ch[pvt->inject.channel][0])
 665                 return -ENODEV;
 666
 667         pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
 668                                 MC_CHANNEL_ERROR_INJECT, 0);
 669
 670         return 0;
 671 }
 672
 673 /*
 674  * i7core inject inject.section
 675  *
 676  *      accept and store error injection inject.section value
 677  *      bit 0 - refers to the lower 32-byte half cacheline
 678  *      bit 1 - refers to the upper 32-byte half cacheline
 679  */
 680 static ssize_t i7core_inject_section_store(struct mem_ctl_info *mci,
 681                                            const char *data, size_t count)
 682 {
 683         struct i7core_pvt *pvt = mci->pvt_info;
 684         unsigned long value;
 685         int rc;
 686
 687         if (pvt->inject.enable)
 688                 disable_inject(mci);
 689
 690         rc = strict_strtoul(data, 10, &value);
 691         if ((rc < 0) || (value > 3))
 692                 return -EIO;
 693
 694         pvt->inject.section = (u32) value;
 695         return count;
 696 }
 697
 698 static ssize_t i7core_inject_section_show(struct mem_ctl_info *mci,
 699                                               char *data)
 700 {
 701         struct i7core_pvt *pvt = mci->pvt_info;
 702         return sprintf(data, "0x%08x\n", pvt->inject.section);
 703 }
 704
 705 /*
 706  * i7core inject.type
 707  *
 708  *      accept and store error injection inject.section value
 709  *      bit 0 - repeat enable - Enable error repetition
 710  *      bit 1 - inject ECC error
 711  *      bit 2 - inject parity error
 712  */
 713 static ssize_t i7core_inject_type_store(struct mem_ctl_info *mci,
 714                                         const char *data, size_t count)
 715 {
 716         struct i7core_pvt *pvt = mci->pvt_info;
 717         unsigned long value;
 718         int rc;
 719
 720         if (pvt->inject.enable)
 721                 disable_inject(mci);
 722
 723         rc = strict_strtoul(data, 10, &value);
 724         if ((rc < 0) || (value > 7))
 725                 return -EIO;
 726
 727         pvt->inject.type = (u32) value;
 728         return count;
 729 }
 730
 731 static ssize_t i7core_inject_type_show(struct mem_ctl_info *mci,
 732                                               char *data)
 733 {
 734         struct i7core_pvt *pvt = mci->pvt_info;
 735         return sprintf(data, "0x%08x\n", pvt->inject.type);
 736 }
 737
 738 /*
 739  * i7core_inject_inject.eccmask_store
 740  *
 741  * The type of error (UE/CE) will depend on the inject.eccmask value:
 742  *   Any bits set to a 1 will flip the corresponding ECC bit
 743  *   Correctable errors can be injected by flipping 1 bit or the bits within
 744  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 745  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 746  *   uncorrectable error to be injected.
 747  */
 748 static ssize_t i7core_inject_eccmask_store(struct mem_ctl_info *mci,
 749                                         const char *data, size_t count)
 750 {
 751         struct i7core_pvt *pvt = mci->pvt_info;
 752         unsigned long value;
 753         int rc;
 754
 755         if (pvt->inject.enable)
 756                 disable_inject(mci);
 757
 758         rc = strict_strtoul(data, 10, &value);
 759         if (rc < 0)
 760                 return -EIO;
 761
 762         pvt->inject.eccmask = (u32) value;
 763         return count;
 764 }
 765
 766 static ssize_t i7core_inject_eccmask_show(struct mem_ctl_info *mci,
 767                                               char *data)
 768 {
 769         struct i7core_pvt *pvt = mci->pvt_info;
 770         return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
 771 }
 772
 773 /*
 774  * i7core_addrmatch
 775  *
 776  * The type of error (UE/CE) will depend on the inject.eccmask value:
 777  *   Any bits set to a 1 will flip the corresponding ECC bit
 778  *   Correctable errors can be injected by flipping 1 bit or the bits within
 779  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 780  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 781  *   uncorrectable error to be injected.
 782  */
 783 static ssize_t i7core_inject_addrmatch_store(struct mem_ctl_info *mci,
 784                                         const char *data, size_t count)
 785 {
 786         struct i7core_pvt *pvt = mci->pvt_info;
 787         char *cmd, *val;
 788         long value;
 789         int rc;
 790
 791         if (pvt->inject.enable)
 792                 disable_inject(mci);
 793
 794         do {
 795                 cmd = strsep((char **) &data, ":");
 796                 if (!cmd)
 797                         break;
 798                 val = strsep((char **) &data, " \n\t");
 799                 if (!val)
 800                         return cmd - data;
 801
 802                 if (!strcasecmp(val, "any"))
 803                         value = -1;
 804                 else {
 805                         rc = strict_strtol(val, 10, &value);
 806                         if ((rc < 0) || (value < 0))
 807                                 return cmd - data;
 808                 }
 809
 810                 if (!strcasecmp(cmd, "channel")) {
 811                         if (value < 3)
 812                                 pvt->inject.channel = value;
 813                         else
 814                                 return cmd - data;
 815                 } else if (!strcasecmp(cmd, "dimm")) {
 816                         if (value < 3)
 817                                 pvt->inject.dimm = value;
 818                         else
 819                                 return cmd - data;
 820                 } else if (!strcasecmp(cmd, "rank")) {
 821                         if (value < 4)
 822                                 pvt->inject.rank = value;
 823                         else
 824                                 return cmd - data;
 825                 } else if (!strcasecmp(cmd, "bank")) {
 826                         if (value < 32)
 827                                 pvt->inject.bank = value;
 828                         else
 829                                 return cmd - data;
 830                 } else if (!strcasecmp(cmd, "page")) {
 831                         if (value <= 0xffff)
 832                                 pvt->inject.page = value;
 833                         else
 834                                 return cmd - data;
 835                 } else if (!strcasecmp(cmd, "col") ||
 836                            !strcasecmp(cmd, "column")) {
 837                         if (value <= 0x3fff)
 838                                 pvt->inject.col = value;
 839                         else
 840                                 return cmd - data;
 841                 }
 842         } while (1);
 843
 844         return count;
 845 }
 846
 847 static ssize_t i7core_inject_addrmatch_show(struct mem_ctl_info *mci,
 848                                               char *data)
 849 {
 850         struct i7core_pvt *pvt = mci->pvt_info;
 851         char channel[4], dimm[4], bank[4], rank[4], page[7], col[7];
 852
 853         if (pvt->inject.channel < 0)
 854                 sprintf(channel, "any");
 855         else
 856                 sprintf(channel, "%d", pvt->inject.channel);
 857         if (pvt->inject.dimm < 0)
 858                 sprintf(dimm, "any");
 859         else
 860                 sprintf(dimm, "%d", pvt->inject.dimm);
 861         if (pvt->inject.bank < 0)
 862                 sprintf(bank, "any");
 863         else
 864                 sprintf(bank, "%d", pvt->inject.bank);
 865         if (pvt->inject.rank < 0)
 866                 sprintf(rank, "any");
 867         else
 868                 sprintf(rank, "%d", pvt->inject.rank);
 869         if (pvt->inject.page < 0)
 870                 sprintf(page, "any");
 871         else
 872                 sprintf(page, "0x%04x", pvt->inject.page);
 873         if (pvt->inject.col < 0)
 874                 sprintf(col, "any");
 875         else
 876                 sprintf(col, "0x%04x", pvt->inject.col);
 877
 878         return sprintf(data, "channel: %s\ndimm: %s\nbank: %s\n"
 879                              "rank: %s\npage: %s\ncolumn: %s\n",
 880                        channel, dimm, bank, rank, page, col);
 881 }
 882
 883 static int write_and_test(struct pci_dev *dev, int where, u32 val)
 884 {
 885         u32 read;
 886         int count;
 887
 888         debugf0("setting pci %02x:%02x.%x reg=%02x value=%08x\n",
 889                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 890                 where, val);
 891
 892         for (count = 0; count < 10; count++) {
 893                 if (count)
 894                         msleep(100);
 895                 pci_write_config_dword(dev, where, val);
 896                 pci_read_config_dword(dev, where, &read);
 897
 898                 if (read == val)
 899                         return 0;
 900         }
 901
 902         i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
 903                 "write=%08x. Read=%08x\n",
 904                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 905                 where, val, read);
 906
 907         return -EINVAL;
 908 }
 909
 910 /*
 911  * This routine prepares the Memory Controller for error injection.
 912  * The error will be injected when some process tries to write to the
 913  * memory that matches the given criteria.
 914  * The criteria can be set in terms of a mask where dimm, rank, bank, page
 915  * and col can be specified.
 916  * A -1 value for any of the mask items will make the MCU to ignore
 917  * that matching criteria for error injection.
 918  *
 919  * It should be noticed that the error will only happen after a write operation
 920  * on a memory that matches the condition. if REPEAT_EN is not enabled at
 921  * inject mask, then it will produce just one error. Otherwise, it will repeat
 922  * until the injectmask would be cleaned.
 923  *
 924  * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
 925  *    is reliable enough to check if the MC is using the
 926  *    three channels. However, this is not clear at the datasheet.
 927  */
 928 static ssize_t i7core_inject_enable_store(struct mem_ctl_info *mci,
 929                                        const char *data, size_t count)
 930 {
 931         struct i7core_pvt *pvt = mci->pvt_info;
 932         u32 injectmask;
 933         u64 mask = 0;
 934         int  rc;
 935         long enable;
 936
 937         if (!pvt->pci_ch[pvt->inject.channel][0])
 938                 return 0;
 939
 940         rc = strict_strtoul(data, 10, &enable);
 941         if ((rc < 0))
 942                 return 0;
 943
 944         if (enable) {
 945                 pvt->inject.enable = 1;
 946         } else {
 947                 disable_inject(mci);
 948                 return count;
 949         }
 950
 951         /* Sets pvt->inject.dimm mask */
 952         if (pvt->inject.dimm < 0)
 953                 mask |= 1L << 41;
 954         else {
 955                 if (pvt->channel[pvt->inject.channel].dimms > 2)
 956                         mask |= (pvt->inject.dimm & 0x3L) << 35;
 957                 else
 958                         mask |= (pvt->inject.dimm & 0x1L) << 36;
 959         }
 960
 961         /* Sets pvt->inject.rank mask */
 962         if (pvt->inject.rank < 0)
 963                 mask |= 1L << 40;
 964         else {
 965                 if (pvt->channel[pvt->inject.channel].dimms > 2)
 966                         mask |= (pvt->inject.rank & 0x1L) << 34;
 967                 else
 968                         mask |= (pvt->inject.rank & 0x3L) << 34;
 969         }
 970
 971         /* Sets pvt->inject.bank mask */
 972         if (pvt->inject.bank < 0)
 973                 mask |= 1L << 39;
 974         else
 975                 mask |= (pvt->inject.bank & 0x15L) << 30;
 976
 977         /* Sets pvt->inject.page mask */
 978         if (pvt->inject.page < 0)
 979                 mask |= 1L << 38;
 980         else
 981                 mask |= (pvt->inject.page & 0xffffL) << 14;
 982
 983         /* Sets pvt->inject.column mask */
 984         if (pvt->inject.col < 0)
 985                 mask |= 1L << 37;
 986         else
 987                 mask |= (pvt->inject.col & 0x3fffL);
 988
 989         /*
 990          * bit    0: REPEAT_EN
 991          * bits 1-2: MASK_HALF_CACHELINE
 992          * bit    3: INJECT_ECC
 993          * bit    4: INJECT_ADDR_PARITY
 994          */
 995
 996         injectmask = (pvt->inject.type & 1) |
 997                      (pvt->inject.section & 0x3) << 1 |
 998                      (pvt->inject.type & 0x6) << (3 - 1);
 999
1000         /* Unlock writes to registers - this register is write only */
1001         pci_write_config_dword(pvt->pci_noncore,
1002                                MC_CFG_CONTROL, 0x2);
1003
1004         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1005                                MC_CHANNEL_ADDR_MATCH, mask);
1006         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1007                                MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
1008
1009         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1010                                MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
1011
1012         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1013                                MC_CHANNEL_ERROR_INJECT, injectmask);
1014
1015         /*
1016          * This is something undocumented, based on my tests
1017          * Without writing 8 to this register, errors aren't injected. Not sure
1018          * why.
1019          */
1020         pci_write_config_dword(pvt->pci_noncore,
1021                                MC_CFG_CONTROL, 8);
1022
1023         debugf0("Error inject addr match 0x%016llx, ecc 0x%08x,"
1024                 " inject 0x%08x\n",
1025                 mask, pvt->inject.eccmask, injectmask);
1026
1027
1028         return count;
1029 }
1030
1031 static ssize_t i7core_inject_enable_show(struct mem_ctl_info *mci,
1032                                         char *data)
1033 {
1034         struct i7core_pvt *pvt = mci->pvt_info;
1035         u32 injectmask;
1036
1037         pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1038                                MC_CHANNEL_ERROR_INJECT, &injectmask);
1039
1040         debugf0("Inject error read: 0x%018x\n", injectmask);
1041
1042         if (injectmask & 0x0c)
1043                 pvt->inject.enable = 1;
1044
1045         return sprintf(data, "%d\n", pvt->inject.enable);
1046 }
1047
1048 static ssize_t i7core_ce_regs_show(struct mem_ctl_info *mci, char *data)
1049 {
1050         unsigned i, count, total = 0;
1051         struct i7core_pvt *pvt = mci->pvt_info;
1052
1053         if (!pvt->ce_count_available) {
1054                 count = sprintf(data, "data unavailable\n");
1055                 return 0;
1056         }
1057         if (!pvt->is_registered) {
1058                 count = sprintf(data, "all channels "
1059                                 "UDIMM0: %lu UDIMM1: %lu UDIMM2: %lu\n",
1060                                 pvt->udimm_ce_count[0],
1061                                 pvt->udimm_ce_count[1],
1062                                 pvt->udimm_ce_count[2]);
1063                 data  += count;
1064                 total += count;
1065         } else {
1066                 for (i = 0; i < NUM_CHANS; i++) {
1067                         count = sprintf(data, "channel %d RDIMM0: %lu "
1068                                         "RDIMM1: %lu RDIMM2: %lu\n",
1069                                         i,
1070                                         pvt->rdimm_ce_count[i][0],
1071                                         pvt->rdimm_ce_count[i][1],
1072                                         pvt->rdimm_ce_count[i][2]);
1073                         data  += count;
1074                         total += count;
1075                 }
1076         }
1077
1078         return total;
1079 }
1080
1081 /*
1082  * Sysfs struct
1083  */
1084 static struct mcidev_sysfs_attribute i7core_inj_attrs[] = {
1085         {
1086                 .attr = {
1087                         .name = "inject_section",
1088                         .mode = (S_IRUGO | S_IWUSR)
1089                 },
1090                 .show  = i7core_inject_section_show,
1091                 .store = i7core_inject_section_store,
1092         }, {
1093                 .attr = {
1094                         .name = "inject_type",
1095                         .mode = (S_IRUGO | S_IWUSR)
1096                 },
1097                 .show  = i7core_inject_type_show,
1098                 .store = i7core_inject_type_store,
1099         }, {
1100                 .attr = {
1101                         .name = "inject_eccmask",
1102                         .mode = (S_IRUGO | S_IWUSR)
1103                 },
1104                 .show  = i7core_inject_eccmask_show,
1105                 .store = i7core_inject_eccmask_store,
1106         }, {
1107                 .attr = {
1108                         .name = "inject_addrmatch",
1109                         .mode = (S_IRUGO | S_IWUSR)
1110                 },
1111                 .show  = i7core_inject_addrmatch_show,
1112                 .store = i7core_inject_addrmatch_store,
1113         }, {
1114                 .attr = {
1115                         .name = "inject_enable",
1116                         .mode = (S_IRUGO | S_IWUSR)
1117                 },
1118                 .show  = i7core_inject_enable_show,
1119                 .store = i7core_inject_enable_store,
1120         }, {
1121                 .attr = {
1122                         .name = "corrected_error_counts",
1123                         .mode = (S_IRUGO | S_IWUSR)
1124                 },
1125                 .show  = i7core_ce_regs_show,
1126                 .store = NULL,
1127         },
1128 };
1129
1130 /****************************************************************************
1131         Device initialization routines: put/get, init/exit
1132  ****************************************************************************/
1133
1134 /*
1135  *      i7core_put_devices      'put' all the devices that we have
1136  *                              reserved via 'get'
1137  */
1138 static void i7core_put_devices(struct i7core_dev *i7core_dev)
1139 {
1140         int i;
1141
1142         for (i = 0; i < N_DEVS; i++)
1143                 pci_dev_put(i7core_dev->pdev[i]);
1144
1145         list_del(&i7core_dev->list);
1146         kfree(i7core_dev->pdev);
1147         kfree(i7core_dev);
1148 }
1149
1150 static void i7core_put_all_devices(void)
1151 {
1152         struct i7core_dev *i7core_dev;
1153
1154         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
1155                 i7core_put_devices(i7core_dev);
1156 }
1157
1158 static void i7core_xeon_pci_fixup(void)
1159 {
1160         struct pci_dev *pdev = NULL;
1161         int i;
1162         /*
1163          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core pci buses
1164          * aren't announced by acpi. So, we need to use a legacy scan probing
1165          * to detect them
1166          */
1167         pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1168                               pci_dev_descr[0].dev_id, NULL);
1169         if (unlikely(!pdev)) {
1170                 for (i = 0; i < MAX_SOCKET_BUSES; i++)
1171                         pcibios_scan_specific_bus(255-i);
1172         }
1173 }
1174
1175 /*
1176  *      i7core_get_devices      Find and perform 'get' operation on the MCH's
1177  *                      device/functions we want to reference for this driver
1178  *
1179  *                      Need to 'get' device 16 func 1 and func 2
1180  */
1181 int i7core_get_onedevice(struct pci_dev **prev, int devno)
1182 {
1183         struct i7core_dev *i7core_dev;
1184
1185         struct pci_dev *pdev = NULL;
1186         u8 bus = 0;
1187         u8 socket = 0;
1188
1189         pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1190                               pci_dev_descr[devno].dev_id, *prev);
1191
1192         /*
1193          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core regs
1194          * is at addr 8086:2c40, instead of 8086:2c41. So, we need
1195          * to probe for the alternate address in case of failure
1196          */
1197         if (pci_dev_descr[devno].dev_id == PCI_DEVICE_ID_INTEL_I7_NOCORE && !pdev)
1198                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1199                                       PCI_DEVICE_ID_INTEL_I7_NOCORE_ALT, *prev);
1200
1201         if (!pdev) {
1202                 if (*prev) {
1203                         *prev = pdev;
1204                         return 0;
1205                 }
1206
1207                 /*
1208                  * Dev 3 function 2 only exists on chips with RDIMMs
1209                  * so, it is ok to not found it
1210                  */
1211                 if ((pci_dev_descr[devno].dev == 3) && (pci_dev_descr[devno].func == 2)) {
1212                         *prev = pdev;
1213                         return 0;
1214                 }
1215
1216                 i7core_printk(KERN_ERR,
1217                         "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1218                         pci_dev_descr[devno].dev, pci_dev_descr[devno].func,
1219                         PCI_VENDOR_ID_INTEL, pci_dev_descr[devno].dev_id);
1220
1221                 /* End of list, leave */
1222                 return -ENODEV;
1223         }
1224         bus = pdev->bus->number;
1225
1226         if (bus == 0x3f)
1227                 socket = 0;
1228         else
1229                 socket = 255 - bus;
1230
1231         i7core_dev = get_i7core_dev(socket);
1232         if (!i7core_dev) {
1233                 i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
1234                 if (!i7core_dev)
1235                         return -ENOMEM;
1236                 i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev) * N_DEVS,
1237                                            GFP_KERNEL);
1238                 if (!i7core_dev->pdev)
1239                         return -ENOMEM;
1240                 i7core_dev->socket = socket;
1241                 list_add_tail(&i7core_dev->list, &i7core_edac_list);
1242         }
1243
1244         if (i7core_dev->pdev[devno]) {
1245                 i7core_printk(KERN_ERR,
1246                         "Duplicated device for "
1247                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1248                         bus, pci_dev_descr[devno].dev, pci_dev_descr[devno].func,
1249                         PCI_VENDOR_ID_INTEL, pci_dev_descr[devno].dev_id);
1250                 pci_dev_put(pdev);
1251                 return -ENODEV;
1252         }
1253
1254         i7core_dev->pdev[devno] = pdev;
1255
1256         /* Sanity check */
1257         if (unlikely(PCI_SLOT(pdev->devfn) != pci_dev_descr[devno].dev ||
1258                         PCI_FUNC(pdev->devfn) != pci_dev_descr[devno].func)) {
1259                 i7core_printk(KERN_ERR,
1260                         "Device PCI ID %04x:%04x "
1261                         "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1262                         PCI_VENDOR_ID_INTEL, pci_dev_descr[devno].dev_id,
1263                         bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1264                         bus, pci_dev_descr[devno].dev, pci_dev_descr[devno].func);
1265                 return -ENODEV;
1266         }
1267
1268         /* Be sure that the device is enabled */
1269         if (unlikely(pci_enable_device(pdev) < 0)) {
1270                 i7core_printk(KERN_ERR,
1271                         "Couldn't enable "
1272                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1273                         bus, pci_dev_descr[devno].dev, pci_dev_descr[devno].func,
1274                         PCI_VENDOR_ID_INTEL, pci_dev_descr[devno].dev_id);
1275                 return -ENODEV;
1276         }
1277
1278         debugf0("Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1279                 socket, bus, pci_dev_descr[devno].dev,
1280                 pci_dev_descr[devno].func,
1281                 PCI_VENDOR_ID_INTEL, pci_dev_descr[devno].dev_id);
1282
1283         *prev = pdev;
1284
1285         return 0;
1286 }
1287
1288 static int i7core_get_devices(void)
1289 {
1290         int i;
1291         struct pci_dev *pdev = NULL;
1292
1293         for (i = 0; i < N_DEVS; i++) {
1294                 pdev = NULL;
1295                 do {
1296                         if (i7core_get_onedevice(&pdev, i) < 0) {
1297                                 i7core_put_all_devices();
1298                                 return -ENODEV;
1299                         }
1300                 } while (pdev);
1301         }
1302
1303         return 0;
1304 }
1305
1306 static int mci_bind_devs(struct mem_ctl_info *mci,
1307                          struct i7core_dev *i7core_dev)
1308 {
1309         struct i7core_pvt *pvt = mci->pvt_info;
1310         struct pci_dev *pdev;
1311         int i, func, slot;
1312
1313         /* Associates i7core_dev and mci for future usage */
1314         pvt->i7core_dev = i7core_dev;
1315         i7core_dev->mci = mci;
1316
1317         pvt->is_registered = 0;
1318         for (i = 0; i < N_DEVS; i++) {
1319                 pdev = i7core_dev->pdev[i];
1320                 if (!pdev)
1321                         continue;
1322
1323                 func = PCI_FUNC(pdev->devfn);
1324                 slot = PCI_SLOT(pdev->devfn);
1325                 if (slot == 3) {
1326                         if (unlikely(func > MAX_MCR_FUNC))
1327                                 goto error;
1328                         pvt->pci_mcr[func] = pdev;
1329                 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1330                         if (unlikely(func > MAX_CHAN_FUNC))
1331                                 goto error;
1332                         pvt->pci_ch[slot - 4][func] = pdev;
1333                 } else if (!slot && !func)
1334                         pvt->pci_noncore = pdev;
1335                 else
1336                         goto error;
1337
1338                 debugf0("Associated fn %d.%d, dev = %p, socket %d\n",
1339                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1340                         pdev, i7core_dev->socket);
1341
1342                 if (PCI_SLOT(pdev->devfn) == 3 &&
1343                         PCI_FUNC(pdev->devfn) == 2)
1344                         pvt->is_registered = 1;
1345         }
1346
1347         return 0;
1348
1349 error:
1350         i7core_printk(KERN_ERR, "Device %d, function %d "
1351                       "is out of the expected range\n",
1352                       slot, func);
1353         return -EINVAL;
1354 }
1355
1356 /****************************************************************************
1357                         Error check routines
1358  ****************************************************************************/
1359 static void i7core_rdimm_update_csrow(struct mem_ctl_info *mci,
1360                                          int chan, int dimm, int add)
1361 {
1362         char *msg;
1363         struct i7core_pvt *pvt = mci->pvt_info;
1364         int row = pvt->csrow_map[chan][dimm], i;
1365
1366         for (i = 0; i < add; i++) {
1367                 msg = kasprintf(GFP_KERNEL, "Corrected error "
1368                                 "(Socket=%d channel=%d dimm=%d)",
1369                                 pvt->i7core_dev->socket, chan, dimm);
1370
1371                 edac_mc_handle_fbd_ce(mci, row, 0, msg);
1372                 kfree (msg);
1373         }
1374 }
1375
1376 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1377                         int chan, int new0, int new1, int new2)
1378 {
1379         struct i7core_pvt *pvt = mci->pvt_info;
1380         int add0 = 0, add1 = 0, add2 = 0;
1381         /* Updates CE counters if it is not the first time here */
1382         if (pvt->ce_count_available) {
1383                 /* Updates CE counters */
1384
1385                 add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1386                 add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1387                 add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1388
1389                 if (add2 < 0)
1390                         add2 += 0x7fff;
1391                 pvt->rdimm_ce_count[chan][2] += add2;
1392
1393                 if (add1 < 0)
1394                         add1 += 0x7fff;
1395                 pvt->rdimm_ce_count[chan][1] += add1;
1396
1397                 if (add0 < 0)
1398                         add0 += 0x7fff;
1399                 pvt->rdimm_ce_count[chan][0] += add0;
1400         } else
1401                 pvt->ce_count_available = 1;
1402
1403         /* Store the new values */
1404         pvt->rdimm_last_ce_count[chan][2] = new2;
1405         pvt->rdimm_last_ce_count[chan][1] = new1;
1406         pvt->rdimm_last_ce_count[chan][0] = new0;
1407
1408         /*updated the edac core */
1409         if (add0 != 0)
1410                 i7core_rdimm_update_csrow(mci, chan, 0, add0);
1411         if (add1 != 0)
1412                 i7core_rdimm_update_csrow(mci, chan, 1, add1);
1413         if (add2 != 0)
1414                 i7core_rdimm_update_csrow(mci, chan, 2, add2);
1415
1416 }
1417
1418 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1419 {
1420         struct i7core_pvt *pvt = mci->pvt_info;
1421         u32 rcv[3][2];
1422         int i, new0, new1, new2;
1423
1424         /*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1425         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1426                                                                 &rcv[0][0]);
1427         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1428                                                                 &rcv[0][1]);
1429         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1430                                                                 &rcv[1][0]);
1431         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1432                                                                 &rcv[1][1]);
1433         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1434                                                                 &rcv[2][0]);
1435         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1436                                                                 &rcv[2][1]);
1437         for (i = 0 ; i < 3; i++) {
1438                 debugf3("MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1439                         (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1440                 /*if the channel has 3 dimms*/
1441                 if (pvt->channel[i].dimms > 2) {
1442                         new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1443                         new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1444                         new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1445                 } else {
1446                         new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1447                                         DIMM_BOT_COR_ERR(rcv[i][0]);
1448                         new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1449                                         DIMM_BOT_COR_ERR(rcv[i][1]);
1450                         new2 = 0;
1451                 }
1452
1453                 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1454         }
1455 }
1456
1457 /* This function is based on the device 3 function 4 registers as described on:
1458  * Intel Xeon Processor 5500 Series Datasheet Volume 2
1459  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1460  * also available at:
1461  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1462  */
1463 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1464 {
1465         struct i7core_pvt *pvt = mci->pvt_info;
1466         u32 rcv1, rcv0;
1467         int new0, new1, new2;
1468
1469         if (!pvt->pci_mcr[4]) {
1470                 debugf0("%s MCR registers not found\n", __func__);
1471                 return;
1472         }
1473
1474         /* Corrected test errors */
1475         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1476         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1477
1478         /* Store the new values */
1479         new2 = DIMM2_COR_ERR(rcv1);
1480         new1 = DIMM1_COR_ERR(rcv0);
1481         new0 = DIMM0_COR_ERR(rcv0);
1482
1483         /* Updates CE counters if it is not the first time here */
1484         if (pvt->ce_count_available) {
1485                 /* Updates CE counters */
1486                 int add0, add1, add2;
1487
1488                 add2 = new2 - pvt->udimm_last_ce_count[2];
1489                 add1 = new1 - pvt->udimm_last_ce_count[1];
1490                 add0 = new0 - pvt->udimm_last_ce_count[0];
1491
1492                 if (add2 < 0)
1493                         add2 += 0x7fff;
1494                 pvt->udimm_ce_count[2] += add2;
1495
1496                 if (add1 < 0)
1497                         add1 += 0x7fff;
1498                 pvt->udimm_ce_count[1] += add1;
1499
1500                 if (add0 < 0)
1501                         add0 += 0x7fff;
1502                 pvt->udimm_ce_count[0] += add0;
1503
1504                 if (add0 | add1 | add2)
1505                         i7core_printk(KERN_ERR, "New Corrected error(s): "
1506                                       "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1507                                       add0, add1, add2);
1508         } else
1509                 pvt->ce_count_available = 1;
1510
1511         /* Store the new values */
1512         pvt->udimm_last_ce_count[2] = new2;
1513         pvt->udimm_last_ce_count[1] = new1;
1514         pvt->udimm_last_ce_count[0] = new0;
1515 }
1516
1517 /*
1518  * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1519  * Architectures Software Developer’s Manual Volume 3B.
1520  * Nehalem are defined as family 0x06, model 0x1a
1521  *
1522  * The MCA registers used here are the following ones:
1523  *     struct mce field MCA Register
1524  *     m->status        MSR_IA32_MC8_STATUS
1525  *     m->addr          MSR_IA32_MC8_ADDR
1526  *     m->misc          MSR_IA32_MC8_MISC
1527  * In the case of Nehalem, the error information is masked at .status and .misc
1528  * fields
1529  */
1530 static void i7core_mce_output_error(struct mem_ctl_info *mci,
1531                                     struct mce *m)
1532 {
1533         struct i7core_pvt *pvt = mci->pvt_info;
1534         char *type, *optype, *err, *msg;
1535         unsigned long error = m->status & 0x1ff0000l;
1536         u32 optypenum = (m->status >> 4) & 0x07;
1537         u32 core_err_cnt = (m->status >> 38) && 0x7fff;
1538         u32 dimm = (m->misc >> 16) & 0x3;
1539         u32 channel = (m->misc >> 18) & 0x3;
1540         u32 syndrome = m->misc >> 32;
1541         u32 errnum = find_first_bit(&error, 32);
1542         int csrow;
1543
1544         if (m->mcgstatus & 1)
1545                 type = "FATAL";
1546         else
1547                 type = "NON_FATAL";
1548
1549         switch (optypenum) {
1550         case 0:
1551                 optype = "generic undef request";
1552                 break;
1553         case 1:
1554                 optype = "read error";
1555                 break;
1556         case 2:
1557                 optype = "write error";
1558                 break;
1559         case 3:
1560                 optype = "addr/cmd error";
1561                 break;
1562         case 4:
1563                 optype = "scrubbing error";
1564                 break;
1565         default:
1566                 optype = "reserved";
1567                 break;
1568         }
1569
1570         switch (errnum) {
1571         case 16:
1572                 err = "read ECC error";
1573                 break;
1574         case 17:
1575                 err = "RAS ECC error";
1576                 break;
1577         case 18:
1578                 err = "write parity error";
1579                 break;
1580         case 19:
1581                 err = "redundacy loss";
1582                 break;
1583         case 20:
1584                 err = "reserved";
1585                 break;
1586         case 21:
1587                 err = "memory range error";
1588                 break;
1589         case 22:
1590                 err = "RTID out of range";
1591                 break;
1592         case 23:
1593                 err = "address parity error";
1594                 break;
1595         case 24:
1596                 err = "byte enable parity error";
1597                 break;
1598         default:
1599                 err = "unknown";
1600         }
1601
1602         /* FIXME: should convert addr into bank and rank information */
1603         msg = kasprintf(GFP_ATOMIC,
1604                 "%s (addr = 0x%08llx, cpu=%d, Dimm=%d, Channel=%d, "
1605                 "syndrome=0x%08x, count=%d, Err=%08llx:%08llx (%s: %s))\n",
1606                 type, (long long) m->addr, m->cpu, dimm, channel,
1607                 syndrome, core_err_cnt, (long long)m->status,
1608                 (long long)m->misc, optype, err);
1609
1610         debugf0("%s", msg);
1611
1612         csrow = pvt->csrow_map[channel][dimm];
1613
1614         /* Call the helper to output message */
1615         if (m->mcgstatus & 1)
1616                 edac_mc_handle_fbd_ue(mci, csrow, 0,
1617                                 0 /* FIXME: should be channel here */, msg);
1618         else if (!pvt->is_registered)
1619                 edac_mc_handle_fbd_ce(mci, csrow,
1620                                 0 /* FIXME: should be channel here */, msg);
1621
1622         kfree(msg);
1623 }
1624
1625 /*
1626  *      i7core_check_error      Retrieve and process errors reported by the
1627  *                              hardware. Called by the Core module.
1628  */
1629 static void i7core_check_error(struct mem_ctl_info *mci)
1630 {
1631         struct i7core_pvt *pvt = mci->pvt_info;
1632         int i;
1633         unsigned count = 0;
1634         struct mce *m = NULL;
1635         unsigned long flags;
1636
1637         /* Copy all mce errors into a temporary buffer */
1638         spin_lock_irqsave(&pvt->mce_lock, flags);
1639         if (pvt->mce_count) {
1640                 m = kmalloc(sizeof(*m) * pvt->mce_count, GFP_ATOMIC);
1641
1642                 if (m) {
1643                         count = pvt->mce_count;
1644                         memcpy(m, &pvt->mce_entry, sizeof(*m) * count);
1645                 }
1646                 pvt->mce_count = 0;
1647         }
1648
1649         spin_unlock_irqrestore(&pvt->mce_lock, flags);
1650
1651         /* proccess mcelog errors */
1652         for (i = 0; i < count; i++)
1653                 i7core_mce_output_error(mci, &m[i]);
1654
1655         kfree(m);
1656
1657         /* check memory count errors */
1658         if (!pvt->is_registered)
1659                 i7core_udimm_check_mc_ecc_err(mci);
1660         else
1661                 i7core_rdimm_check_mc_ecc_err(mci);
1662 }
1663
1664 /*
1665  * i7core_mce_check_error       Replicates mcelog routine to get errors
1666  *                              This routine simply queues mcelog errors, and
1667  *                              return. The error itself should be handled later
1668  *                              by i7core_check_error.
1669  */
1670 static int i7core_mce_check_error(void *priv, struct mce *mce)
1671 {
1672         struct mem_ctl_info *mci = priv;
1673         struct i7core_pvt *pvt = mci->pvt_info;
1674         unsigned long flags;
1675
1676         /*
1677          * Just let mcelog handle it if the error is
1678          * outside the memory controller
1679          */
1680         if (((mce->status & 0xffff) >> 7) != 1)
1681                 return 0;
1682
1683         /* Bank 8 registers are the only ones that we know how to handle */
1684         if (mce->bank != 8)
1685                 return 0;
1686
1687         /* Only handle if it is the right mc controller */
1688         if (cpu_data(mce->cpu).phys_proc_id != pvt->i7core_dev->socket) {
1689                 debugf0("mc%d: ignoring mce log for socket %d. "
1690                         "Another mc should get it.\n",
1691                         pvt->i7core_dev->socket,
1692                         cpu_data(mce->cpu).phys_proc_id);
1693                 return 0;
1694         }
1695
1696         spin_lock_irqsave(&pvt->mce_lock, flags);
1697         if (pvt->mce_count < MCE_LOG_LEN) {
1698                 memcpy(&pvt->mce_entry[pvt->mce_count], mce, sizeof(*mce));
1699                 pvt->mce_count++;
1700         }
1701         spin_unlock_irqrestore(&pvt->mce_lock, flags);
1702
1703         /* Handle fatal errors immediately */
1704         if (mce->mcgstatus & 1)
1705                 i7core_check_error(mci);
1706
1707         /* Advice mcelog that the error were handled */
1708         return 1;
1709 }
1710
1711 static int i7core_register_mci(struct i7core_dev *i7core_dev,
1712                                int num_channels, int num_csrows)
1713 {
1714         struct mem_ctl_info *mci;
1715         struct i7core_pvt *pvt;
1716         int csrow = 0;
1717         int rc;
1718
1719         /* allocate a new MC control structure */
1720         mci = edac_mc_alloc(sizeof(*pvt), num_csrows, num_channels,
1721                             i7core_dev->socket);
1722         if (unlikely(!mci))
1723                 return -ENOMEM;
1724
1725         debugf0("MC: " __FILE__ ": %s(): mci = %p\n", __func__, mci);
1726
1727         /* record ptr to the generic device */
1728         mci->dev = &i7core_dev->pdev[0]->dev;
1729
1730         pvt = mci->pvt_info;
1731         memset(pvt, 0, sizeof(*pvt));
1732
1733         /*
1734          * FIXME: how to handle RDDR3 at MCI level? It is possible to have
1735          * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
1736          * memory channels
1737          */
1738         mci->mtype_cap = MEM_FLAG_DDR3;
1739         mci->edac_ctl_cap = EDAC_FLAG_NONE;
1740         mci->edac_cap = EDAC_FLAG_NONE;
1741         mci->mod_name = "i7core_edac.c";
1742         mci->mod_ver = I7CORE_REVISION;
1743         mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
1744                                   i7core_dev->socket);
1745         mci->dev_name = pci_name(i7core_dev->pdev[0]);
1746         mci->ctl_page_to_phys = NULL;
1747         mci->mc_driver_sysfs_attributes = i7core_inj_attrs;
1748         /* Set the function pointer to an actual operation function */
1749         mci->edac_check = i7core_check_error;
1750
1751         /* Store pci devices at mci for faster access */
1752         rc = mci_bind_devs(mci, i7core_dev);
1753         if (unlikely(rc < 0))
1754                 goto fail;
1755
1756         /* Get dimm basic config */
1757         get_dimm_config(mci, &csrow);
1758
1759         /* add this new MC control structure to EDAC's list of MCs */
1760         if (unlikely(edac_mc_add_mc(mci))) {
1761                 debugf0("MC: " __FILE__
1762                         ": %s(): failed edac_mc_add_mc()\n", __func__);
1763                 /* FIXME: perhaps some code should go here that disables error
1764                  * reporting if we just enabled it
1765                  */
1766
1767                 rc = -EINVAL;
1768                 goto fail;
1769         }
1770
1771         /* allocating generic PCI control info */
1772         i7core_pci = edac_pci_create_generic_ctl(&i7core_dev->pdev[0]->dev,
1773                                                  EDAC_MOD_STR);
1774         if (unlikely(!i7core_pci)) {
1775                 printk(KERN_WARNING
1776                         "%s(): Unable to create PCI control\n",
1777                         __func__);
1778                 printk(KERN_WARNING
1779                         "%s(): PCI error report via EDAC not setup\n",
1780                         __func__);
1781         }
1782
1783         /* Default error mask is any memory */
1784         pvt->inject.channel = 0;
1785         pvt->inject.dimm = -1;
1786         pvt->inject.rank = -1;
1787         pvt->inject.bank = -1;
1788         pvt->inject.page = -1;
1789         pvt->inject.col = -1;
1790
1791         /* Registers on edac_mce in order to receive memory errors */
1792         pvt->edac_mce.priv = mci;
1793         pvt->edac_mce.check_error = i7core_mce_check_error;
1794         spin_lock_init(&pvt->mce_lock);
1795
1796         rc = edac_mce_register(&pvt->edac_mce);
1797         if (unlikely(rc < 0)) {
1798                 debugf0("MC: " __FILE__
1799                         ": %s(): failed edac_mce_register()\n", __func__);
1800         }
1801
1802 fail:
1803         edac_mc_free(mci);
1804         return rc;
1805 }
1806
1807 /*
1808  *      i7core_probe    Probe for ONE instance of device to see if it is
1809  *                      present.
1810  *      return:
1811  *              0 for FOUND a device
1812  *              < 0 for error code
1813  */
1814 static int __devinit i7core_probe(struct pci_dev *pdev,
1815                                   const struct pci_device_id *id)
1816 {
1817         int dev_idx = id->driver_data;
1818         int rc;
1819         struct i7core_dev *i7core_dev;
1820
1821         /*
1822          * All memory controllers are allocated at the first pass.
1823          */
1824         if (unlikely(dev_idx >= 1))
1825                 return -EINVAL;
1826
1827         /* get the pci devices we want to reserve for our use */
1828         mutex_lock(&i7core_edac_lock);
1829         rc = i7core_get_devices();
1830         if (unlikely(rc < 0))
1831                 goto fail0;
1832
1833         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
1834                 int channels;
1835                 int csrows;
1836
1837                 /* Check the number of active and not disabled channels */
1838                 rc = i7core_get_active_channels(i7core_dev->socket,
1839                                                 &channels, &csrows);
1840                 if (unlikely(rc < 0))
1841                         goto fail1;
1842
1843                 rc = i7core_register_mci(i7core_dev, channels, csrows);
1844                 if (unlikely(rc < 0))
1845                         goto fail1;
1846         }
1847
1848         i7core_printk(KERN_INFO, "Driver loaded.\n");
1849
1850         mutex_unlock(&i7core_edac_lock);
1851         return 0;
1852
1853 fail1:
1854         i7core_put_all_devices();
1855 fail0:
1856         mutex_unlock(&i7core_edac_lock);
1857         return rc;
1858 }
1859
1860 /*
1861  *      i7core_remove   destructor for one instance of device
1862  *
1863  */
1864 static void __devexit i7core_remove(struct pci_dev *pdev)
1865 {
1866         struct mem_ctl_info *mci;
1867         struct i7core_pvt *pvt;
1868         struct i7core_dev *i7core_dev;
1869
1870         debugf0(__FILE__ ": %s()\n", __func__);
1871
1872         if (i7core_pci)
1873                 edac_pci_release_generic_ctl(i7core_pci);
1874
1875
1876         mci = edac_mc_del_mc(&pdev->dev);
1877         if (!mci)
1878                 return;
1879
1880         /* Unregisters on edac_mce in order to receive memory errors */
1881         pvt = mci->pvt_info;
1882         i7core_dev = pvt->i7core_dev;
1883         edac_mce_unregister(&pvt->edac_mce);
1884
1885         /* retrieve references to resources, and free those resources */
1886         mutex_lock(&i7core_edac_lock);
1887         i7core_put_devices(i7core_dev);
1888         mutex_unlock(&i7core_edac_lock);
1889
1890         kfree(mci->ctl_name);
1891         edac_mc_free(mci);
1892 }
1893
1894 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
1895
1896 /*
1897  *      i7core_driver   pci_driver structure for this module
1898  *
1899  */
1900 static struct pci_driver i7core_driver = {
1901         .name     = "i7core_edac",
1902         .probe    = i7core_probe,
1903         .remove   = __devexit_p(i7core_remove),
1904         .id_table = i7core_pci_tbl,
1905 };
1906
1907 /*
1908  *      i7core_init             Module entry function
1909  *                      Try to initialize this module for its devices
1910  */
1911 static int __init i7core_init(void)
1912 {
1913         int pci_rc;
1914
1915         debugf2("MC: " __FILE__ ": %s()\n", __func__);
1916
1917         /* Ensure that the OPSTATE is set correctly for POLL or NMI */
1918         opstate_init();
1919
1920         i7core_xeon_pci_fixup();
1921
1922         pci_rc = pci_register_driver(&i7core_driver);
1923
1924         if (pci_rc >= 0)
1925                 return 0;
1926
1927         i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
1928                       pci_rc);
1929
1930         return pci_rc;
1931 }
1932
1933 /*
1934  *      i7core_exit()   Module exit function
1935  *                      Unregister the driver
1936  */
1937 static void __exit i7core_exit(void)
1938 {
1939         debugf2("MC: " __FILE__ ": %s()\n", __func__);
1940         pci_unregister_driver(&i7core_driver);
1941 }
1942
1943 module_init(i7core_init);
1944 module_exit(i7core_exit);
1945
1946 MODULE_LICENSE("GPL");
1947 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
1948 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
1949 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
1950                    I7CORE_REVISION);
1951
1952 module_param(edac_op_state, int, 0444);
1953 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");