drivers/edac/i7core_edac.c

   1 /* Intel i7 core/Nehalem Memory Controller kernel module
   2  *
   3  * This driver supports the memory controllers found on the Intel
   4  * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
   5  * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
   6  * and Westmere-EP.
   7  *
   8  * This file may be distributed under the terms of the
   9  * GNU General Public License version 2 only.
  10  *
  11  * Copyright (c) 2009-2010 by:
  12  *       Mauro Carvalho Chehab <mchehab@redhat.com>
  13  *
  14  * Red Hat Inc. http://www.redhat.com
  15  *
  16  * Forked and adapted from the i5400_edac driver
  17  *
  18  * Based on the following public Intel datasheets:
  19  * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
  20  * Datasheet, Volume 2:
  21  *      http://download.intel.com/design/processor/datashts/320835.pdf
  22  * Intel Xeon Processor 5500 Series Datasheet Volume 2
  23  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
  24  * also available at:
  25  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/init.h>
  30 #include <linux/pci.h>
  31 #include <linux/pci_ids.h>
  32 #include <linux/slab.h>
  33 #include <linux/delay.h>
  34 #include <linux/dmi.h>
  35 #include <linux/edac.h>
  36 #include <linux/mmzone.h>
  37 #include <linux/smp.h>
  38 #include <asm/mce.h>
  39 #include <asm/processor.h>
  40 #include <asm/div64.h>
  41
  42 #include "edac_core.h"
  43
  44 /* Static vars */
  45 static LIST_HEAD(i7core_edac_list);
  46 static DEFINE_MUTEX(i7core_edac_lock);
  47 static int probed;
  48
  49 static int use_pci_fixup;
  50 module_param(use_pci_fixup, int, 0444);
  51 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
  52 /*
  53  * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
  54  * registers start at bus 255, and are not reported by BIOS.
  55  * We currently find devices with only 2 sockets. In order to support more QPI
  56  * Quick Path Interconnect, just increment this number.
  57  */
  58 #define MAX_SOCKET_BUSES        2
  59
  60
  61 /*
  62  * Alter this version for the module when modifications are made
  63  */
  64 #define I7CORE_REVISION    " Ver: 1.0.0"
  65 #define EDAC_MOD_STR      "i7core_edac"
  66
  67 /*
  68  * Debug macros
  69  */
  70 #define i7core_printk(level, fmt, arg...)                       \
  71         edac_printk(level, "i7core", fmt, ##arg)
  72
  73 #define i7core_mc_printk(mci, level, fmt, arg...)               \
  74         edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
  75
  76 /*
  77  * i7core Memory Controller Registers
  78  */
  79
  80         /* OFFSETS for Device 0 Function 0 */
  81
  82 #define MC_CFG_CONTROL  0x90
  83   #define MC_CFG_UNLOCK         0x02
  84   #define MC_CFG_LOCK           0x00
  85
  86         /* OFFSETS for Device 3 Function 0 */
  87
  88 #define MC_CONTROL      0x48
  89 #define MC_STATUS       0x4c
  90 #define MC_MAX_DOD      0x64
  91
  92 /*
  93  * OFFSETS for Device 3 Function 4, as inicated on Xeon 5500 datasheet:
  94  * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  95  */
  96
  97 #define MC_TEST_ERR_RCV1        0x60
  98   #define DIMM2_COR_ERR(r)                      ((r) & 0x7fff)
  99
 100 #define MC_TEST_ERR_RCV0        0x64
 101   #define DIMM1_COR_ERR(r)                      (((r) >> 16) & 0x7fff)
 102   #define DIMM0_COR_ERR(r)                      ((r) & 0x7fff)
 103
 104 /* OFFSETS for Device 3 Function 2, as inicated on Xeon 5500 datasheet */
 105 #define MC_SSRCONTROL           0x48
 106   #define SSR_MODE_DISABLE      0x00
 107   #define SSR_MODE_ENABLE       0x01
 108   #define SSR_MODE_MASK         0x03
 109
 110 #define MC_SCRUB_CONTROL        0x4c
 111   #define STARTSCRUB            (1 << 24)
 112   #define SCRUBINTERVAL_MASK    0xffffff
 113
 114 #define MC_COR_ECC_CNT_0        0x80
 115 #define MC_COR_ECC_CNT_1        0x84
 116 #define MC_COR_ECC_CNT_2        0x88
 117 #define MC_COR_ECC_CNT_3        0x8c
 118 #define MC_COR_ECC_CNT_4        0x90
 119 #define MC_COR_ECC_CNT_5        0x94
 120
 121 #define DIMM_TOP_COR_ERR(r)                     (((r) >> 16) & 0x7fff)
 122 #define DIMM_BOT_COR_ERR(r)                     ((r) & 0x7fff)
 123
 124
 125         /* OFFSETS for Devices 4,5 and 6 Function 0 */
 126
 127 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
 128   #define THREE_DIMMS_PRESENT           (1 << 24)
 129   #define SINGLE_QUAD_RANK_PRESENT      (1 << 23)
 130   #define QUAD_RANK_PRESENT             (1 << 22)
 131   #define REGISTERED_DIMM               (1 << 15)
 132
 133 #define MC_CHANNEL_MAPPER       0x60
 134   #define RDLCH(r, ch)          ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
 135   #define WRLCH(r, ch)          ((((r) >> (ch * 6)) & 0x07) - 1)
 136
 137 #define MC_CHANNEL_RANK_PRESENT 0x7c
 138   #define RANK_PRESENT_MASK             0xffff
 139
 140 #define MC_CHANNEL_ADDR_MATCH   0xf0
 141 #define MC_CHANNEL_ERROR_MASK   0xf8
 142 #define MC_CHANNEL_ERROR_INJECT 0xfc
 143   #define INJECT_ADDR_PARITY    0x10
 144   #define INJECT_ECC            0x08
 145   #define MASK_CACHELINE        0x06
 146   #define MASK_FULL_CACHELINE   0x06
 147   #define MASK_MSB32_CACHELINE  0x04
 148   #define MASK_LSB32_CACHELINE  0x02
 149   #define NO_MASK_CACHELINE     0x00
 150   #define REPEAT_EN             0x01
 151
 152         /* OFFSETS for Devices 4,5 and 6 Function 1 */
 153
 154 #define MC_DOD_CH_DIMM0         0x48
 155 #define MC_DOD_CH_DIMM1         0x4c
 156 #define MC_DOD_CH_DIMM2         0x50
 157   #define RANKOFFSET_MASK       ((1 << 12) | (1 << 11) | (1 << 10))
 158   #define RANKOFFSET(x)         ((x & RANKOFFSET_MASK) >> 10)
 159   #define DIMM_PRESENT_MASK     (1 << 9)
 160   #define DIMM_PRESENT(x)       (((x) & DIMM_PRESENT_MASK) >> 9)
 161   #define MC_DOD_NUMBANK_MASK           ((1 << 8) | (1 << 7))
 162   #define MC_DOD_NUMBANK(x)             (((x) & MC_DOD_NUMBANK_MASK) >> 7)
 163   #define MC_DOD_NUMRANK_MASK           ((1 << 6) | (1 << 5))
 164   #define MC_DOD_NUMRANK(x)             (((x) & MC_DOD_NUMRANK_MASK) >> 5)
 165   #define MC_DOD_NUMROW_MASK            ((1 << 4) | (1 << 3) | (1 << 2))
 166   #define MC_DOD_NUMROW(x)              (((x) & MC_DOD_NUMROW_MASK) >> 2)
 167   #define MC_DOD_NUMCOL_MASK            3
 168   #define MC_DOD_NUMCOL(x)              ((x) & MC_DOD_NUMCOL_MASK)
 169
 170 #define MC_RANK_PRESENT         0x7c
 171
 172 #define MC_SAG_CH_0     0x80
 173 #define MC_SAG_CH_1     0x84
 174 #define MC_SAG_CH_2     0x88
 175 #define MC_SAG_CH_3     0x8c
 176 #define MC_SAG_CH_4     0x90
 177 #define MC_SAG_CH_5     0x94
 178 #define MC_SAG_CH_6     0x98
 179 #define MC_SAG_CH_7     0x9c
 180
 181 #define MC_RIR_LIMIT_CH_0       0x40
 182 #define MC_RIR_LIMIT_CH_1       0x44
 183 #define MC_RIR_LIMIT_CH_2       0x48
 184 #define MC_RIR_LIMIT_CH_3       0x4C
 185 #define MC_RIR_LIMIT_CH_4       0x50
 186 #define MC_RIR_LIMIT_CH_5       0x54
 187 #define MC_RIR_LIMIT_CH_6       0x58
 188 #define MC_RIR_LIMIT_CH_7       0x5C
 189 #define MC_RIR_LIMIT_MASK       ((1 << 10) - 1)
 190
 191 #define MC_RIR_WAY_CH           0x80
 192   #define MC_RIR_WAY_OFFSET_MASK        (((1 << 14) - 1) & ~0x7)
 193   #define MC_RIR_WAY_RANK_MASK          0x7
 194
 195 /*
 196  * i7core structs
 197  */
 198
 199 #define NUM_CHANS 3
 200 #define MAX_DIMMS 3             /* Max DIMMS per channel */
 201 #define MAX_MCR_FUNC  4
 202 #define MAX_CHAN_FUNC 3
 203
 204 struct i7core_info {
 205         u32     mc_control;
 206         u32     mc_status;
 207         u32     max_dod;
 208         u32     ch_map;
 209 };
 210
 211
 212 struct i7core_inject {
 213         int     enable;
 214
 215         u32     section;
 216         u32     type;
 217         u32     eccmask;
 218
 219         /* Error address mask */
 220         int channel, dimm, rank, bank, page, col;
 221 };
 222
 223 struct i7core_channel {
 224         u32             ranks;
 225         u32             dimms;
 226 };
 227
 228 struct pci_id_descr {
 229         int                     dev;
 230         int                     func;
 231         int                     dev_id;
 232         int                     optional;
 233 };
 234
 235 struct pci_id_table {
 236         const struct pci_id_descr       *descr;
 237         int                             n_devs;
 238 };
 239
 240 struct i7core_dev {
 241         struct list_head        list;
 242         u8                      socket;
 243         struct pci_dev          **pdev;
 244         int                     n_devs;
 245         struct mem_ctl_info     *mci;
 246 };
 247
 248 struct i7core_pvt {
 249         struct pci_dev  *pci_noncore;
 250         struct pci_dev  *pci_mcr[MAX_MCR_FUNC + 1];
 251         struct pci_dev  *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
 252
 253         struct i7core_dev *i7core_dev;
 254
 255         struct i7core_info      info;
 256         struct i7core_inject    inject;
 257         struct i7core_channel   channel[NUM_CHANS];
 258
 259         int             ce_count_available;
 260         int             csrow_map[NUM_CHANS][MAX_DIMMS];
 261
 262                         /* ECC corrected errors counts per udimm */
 263         unsigned long   udimm_ce_count[MAX_DIMMS];
 264         int             udimm_last_ce_count[MAX_DIMMS];
 265                         /* ECC corrected errors counts per rdimm */
 266         unsigned long   rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
 267         int             rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
 268
 269         bool            is_registered, enable_scrub;
 270
 271         /* Fifo double buffers */
 272         struct mce              mce_entry[MCE_LOG_LEN];
 273         struct mce              mce_outentry[MCE_LOG_LEN];
 274
 275         /* Fifo in/out counters */
 276         unsigned                mce_in, mce_out;
 277
 278         /* Count indicator to show errors not got */
 279         unsigned                mce_overrun;
 280
 281         /* DCLK Frequency used for computing scrub rate */
 282         int                     dclk_freq;
 283
 284         /* Struct to control EDAC polling */
 285         struct edac_pci_ctl_info *i7core_pci;
 286 };
 287
 288 #define PCI_DESCR(device, function, device_id)  \
 289         .dev = (device),                        \
 290         .func = (function),                     \
 291         .dev_id = (device_id)
 292
 293 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
 294                 /* Memory controller */
 295         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
 296         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
 297                         /* Exists only for RDIMM */
 298         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1  },
 299         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
 300
 301                 /* Channel 0 */
 302         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
 303         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
 304         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
 305         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
 306
 307                 /* Channel 1 */
 308         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
 309         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
 310         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
 311         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
 312
 313                 /* Channel 2 */
 314         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
 315         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
 316         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
 317         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
 318
 319                 /* Generic Non-core registers */
 320         /*
 321          * This is the PCI device on i7core and on Xeon 35xx (8086:2c41)
 322          * On Xeon 55xx, however, it has a different id (8086:2c40). So,
 323          * the probing code needs to test for the other address in case of
 324          * failure of this one
 325          */
 326         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE)  },
 327
 328 };
 329
 330 static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
 331         { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR)         },
 332         { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD)      },
 333         { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST)     },
 334
 335         { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
 336         { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
 337         { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
 338         { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC)   },
 339
 340         { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
 341         { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
 342         { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
 343         { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC)   },
 344
 345         /*
 346          * This is the PCI device has an alternate address on some
 347          * processors like Core i7 860
 348          */
 349         { PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE)     },
 350 };
 351
 352 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
 353                 /* Memory controller */
 354         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2)     },
 355         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2)  },
 356                         /* Exists only for RDIMM */
 357         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1  },
 358         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
 359
 360                 /* Channel 0 */
 361         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
 362         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
 363         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
 364         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2)   },
 365
 366                 /* Channel 1 */
 367         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
 368         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
 369         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
 370         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2)   },
 371
 372                 /* Channel 2 */
 373         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
 374         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
 375         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
 376         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2)   },
 377
 378                 /* Generic Non-core registers */
 379         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2)  },
 380
 381 };
 382
 383 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 384 static const struct pci_id_table pci_dev_table[] = {
 385         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
 386         PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
 387         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
 388         {0,}                    /* 0 terminated list. */
 389 };
 390
 391 /*
 392  *      pci_device_id   table for which devices we are looking for
 393  */
 394 static const struct pci_device_id i7core_pci_tbl[] __devinitdata = {
 395         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
 396         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
 397         {0,}                    /* 0 terminated list. */
 398 };
 399
 400 /****************************************************************************
 401                         Anciliary status routines
 402  ****************************************************************************/
 403
 404         /* MC_CONTROL bits */
 405 #define CH_ACTIVE(pvt, ch)      ((pvt)->info.mc_control & (1 << (8 + ch)))
 406 #define ECCx8(pvt)              ((pvt)->info.mc_control & (1 << 1))
 407
 408         /* MC_STATUS bits */
 409 #define ECC_ENABLED(pvt)        ((pvt)->info.mc_status & (1 << 4))
 410 #define CH_DISABLED(pvt, ch)    ((pvt)->info.mc_status & (1 << ch))
 411
 412         /* MC_MAX_DOD read functions */
 413 static inline int numdimms(u32 dimms)
 414 {
 415         return (dimms & 0x3) + 1;
 416 }
 417
 418 static inline int numrank(u32 rank)
 419 {
 420         static int ranks[4] = { 1, 2, 4, -EINVAL };
 421
 422         return ranks[rank & 0x3];
 423 }
 424
 425 static inline int numbank(u32 bank)
 426 {
 427         static int banks[4] = { 4, 8, 16, -EINVAL };
 428
 429         return banks[bank & 0x3];
 430 }
 431
 432 static inline int numrow(u32 row)
 433 {
 434         static int rows[8] = {
 435                 1 << 12, 1 << 13, 1 << 14, 1 << 15,
 436                 1 << 16, -EINVAL, -EINVAL, -EINVAL,
 437         };
 438
 439         return rows[row & 0x7];
 440 }
 441
 442 static inline int numcol(u32 col)
 443 {
 444         static int cols[8] = {
 445                 1 << 10, 1 << 11, 1 << 12, -EINVAL,
 446         };
 447         return cols[col & 0x3];
 448 }
 449
 450 static struct i7core_dev *get_i7core_dev(u8 socket)
 451 {
 452         struct i7core_dev *i7core_dev;
 453
 454         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
 455                 if (i7core_dev->socket == socket)
 456                         return i7core_dev;
 457         }
 458
 459         return NULL;
 460 }
 461
 462 static struct i7core_dev *alloc_i7core_dev(u8 socket,
 463                                            const struct pci_id_table *table)
 464 {
 465         struct i7core_dev *i7core_dev;
 466
 467         i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
 468         if (!i7core_dev)
 469                 return NULL;
 470
 471         i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev) * table->n_devs,
 472                                    GFP_KERNEL);
 473         if (!i7core_dev->pdev) {
 474                 kfree(i7core_dev);
 475                 return NULL;
 476         }
 477
 478         i7core_dev->socket = socket;
 479         i7core_dev->n_devs = table->n_devs;
 480         list_add_tail(&i7core_dev->list, &i7core_edac_list);
 481
 482         return i7core_dev;
 483 }
 484
 485 static void free_i7core_dev(struct i7core_dev *i7core_dev)
 486 {
 487         list_del(&i7core_dev->list);
 488         kfree(i7core_dev->pdev);
 489         kfree(i7core_dev);
 490 }
 491
 492 /****************************************************************************
 493                         Memory check routines
 494  ****************************************************************************/
 495 static struct pci_dev *get_pdev_slot_func(u8 socket, unsigned slot,
 496                                           unsigned func)
 497 {
 498         struct i7core_dev *i7core_dev = get_i7core_dev(socket);
 499         int i;
 500
 501         if (!i7core_dev)
 502                 return NULL;
 503
 504         for (i = 0; i < i7core_dev->n_devs; i++) {
 505                 if (!i7core_dev->pdev[i])
 506                         continue;
 507
 508                 if (PCI_SLOT(i7core_dev->pdev[i]->devfn) == slot &&
 509                     PCI_FUNC(i7core_dev->pdev[i]->devfn) == func) {
 510                         return i7core_dev->pdev[i];
 511                 }
 512         }
 513
 514         return NULL;
 515 }
 516
 517 /**
 518  * i7core_get_active_channels() - gets the number of channels and csrows
 519  * @socket:     Quick Path Interconnect socket
 520  * @channels:   Number of channels that will be returned
 521  * @csrows:     Number of csrows found
 522  *
 523  * Since EDAC core needs to know in advance the number of available channels
 524  * and csrows, in order to allocate memory for csrows/channels, it is needed
 525  * to run two similar steps. At the first step, implemented on this function,
 526  * it checks the number of csrows/channels present at one socket.
 527  * this is used in order to properly allocate the size of mci components.
 528  *
 529  * It should be noticed that none of the current available datasheets explain
 530  * or even mention how csrows are seen by the memory controller. So, we need
 531  * to add a fake description for csrows.
 532  * So, this driver is attributing one DIMM memory for one csrow.
 533  */
 534 static int i7core_get_active_channels(const u8 socket, unsigned *channels,
 535                                       unsigned *csrows)
 536 {
 537         struct pci_dev *pdev = NULL;
 538         int i, j;
 539         u32 status, control;
 540
 541         *channels = 0;
 542         *csrows = 0;
 543
 544         pdev = get_pdev_slot_func(socket, 3, 0);
 545         if (!pdev) {
 546                 i7core_printk(KERN_ERR, "Couldn't find socket %d fn 3.0!!!\n",
 547                               socket);
 548                 return -ENODEV;
 549         }
 550
 551         /* Device 3 function 0 reads */
 552         pci_read_config_dword(pdev, MC_STATUS, &status);
 553         pci_read_config_dword(pdev, MC_CONTROL, &control);
 554
 555         for (i = 0; i < NUM_CHANS; i++) {
 556                 u32 dimm_dod[3];
 557                 /* Check if the channel is active */
 558                 if (!(control & (1 << (8 + i))))
 559                         continue;
 560
 561                 /* Check if the channel is disabled */
 562                 if (status & (1 << i))
 563                         continue;
 564
 565                 pdev = get_pdev_slot_func(socket, i + 4, 1);
 566                 if (!pdev) {
 567                         i7core_printk(KERN_ERR, "Couldn't find socket %d "
 568                                                 "fn %d.%d!!!\n",
 569                                                 socket, i + 4, 1);
 570                         return -ENODEV;
 571                 }
 572                 /* Devices 4-6 function 1 */
 573                 pci_read_config_dword(pdev,
 574                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 575                 pci_read_config_dword(pdev,
 576                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 577                 pci_read_config_dword(pdev,
 578                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 579
 580                 (*channels)++;
 581
 582                 for (j = 0; j < 3; j++) {
 583                         if (!DIMM_PRESENT(dimm_dod[j]))
 584                                 continue;
 585                         (*csrows)++;
 586                 }
 587         }
 588
 589         debugf0("Number of active channels on socket %d: %d\n",
 590                 socket, *channels);
 591
 592         return 0;
 593 }
 594
 595 static int get_dimm_config(const struct mem_ctl_info *mci)
 596 {
 597         struct i7core_pvt *pvt = mci->pvt_info;
 598         struct csrow_info *csr;
 599         struct pci_dev *pdev;
 600         int i, j;
 601         int csrow = 0;
 602         unsigned long last_page = 0;
 603         enum edac_type mode;
 604         enum mem_type mtype;
 605
 606         /* Get data from the MC register, function 0 */
 607         pdev = pvt->pci_mcr[0];
 608         if (!pdev)
 609                 return -ENODEV;
 610
 611         /* Device 3 function 0 reads */
 612         pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
 613         pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
 614         pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
 615         pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
 616
 617         debugf0("QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
 618                 pvt->i7core_dev->socket, pvt->info.mc_control, pvt->info.mc_status,
 619                 pvt->info.max_dod, pvt->info.ch_map);
 620
 621         if (ECC_ENABLED(pvt)) {
 622                 debugf0("ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
 623                 if (ECCx8(pvt))
 624                         mode = EDAC_S8ECD8ED;
 625                 else
 626                         mode = EDAC_S4ECD4ED;
 627         } else {
 628                 debugf0("ECC disabled\n");
 629                 mode = EDAC_NONE;
 630         }
 631
 632         /* FIXME: need to handle the error codes */
 633         debugf0("DOD Max limits: DIMMS: %d, %d-ranked, %d-banked "
 634                 "x%x x 0x%x\n",
 635                 numdimms(pvt->info.max_dod),
 636                 numrank(pvt->info.max_dod >> 2),
 637                 numbank(pvt->info.max_dod >> 4),
 638                 numrow(pvt->info.max_dod >> 6),
 639                 numcol(pvt->info.max_dod >> 9));
 640
 641         for (i = 0; i < NUM_CHANS; i++) {
 642                 u32 data, dimm_dod[3], value[8];
 643
 644                 if (!pvt->pci_ch[i][0])
 645                         continue;
 646
 647                 if (!CH_ACTIVE(pvt, i)) {
 648                         debugf0("Channel %i is not active\n", i);
 649                         continue;
 650                 }
 651                 if (CH_DISABLED(pvt, i)) {
 652                         debugf0("Channel %i is disabled\n", i);
 653                         continue;
 654                 }
 655
 656                 /* Devices 4-6 function 0 */
 657                 pci_read_config_dword(pvt->pci_ch[i][0],
 658                                 MC_CHANNEL_DIMM_INIT_PARAMS, &data);
 659
 660                 pvt->channel[i].ranks = (data & QUAD_RANK_PRESENT) ?
 661                                                 4 : 2;
 662
 663                 if (data & REGISTERED_DIMM)
 664                         mtype = MEM_RDDR3;
 665                 else
 666                         mtype = MEM_DDR3;
 667 #if 0
 668                 if (data & THREE_DIMMS_PRESENT)
 669                         pvt->channel[i].dimms = 3;
 670                 else if (data & SINGLE_QUAD_RANK_PRESENT)
 671                         pvt->channel[i].dimms = 1;
 672                 else
 673                         pvt->channel[i].dimms = 2;
 674 #endif
 675
 676                 /* Devices 4-6 function 1 */
 677                 pci_read_config_dword(pvt->pci_ch[i][1],
 678                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 679                 pci_read_config_dword(pvt->pci_ch[i][1],
 680                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 681                 pci_read_config_dword(pvt->pci_ch[i][1],
 682                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 683
 684                 debugf0("Ch%d phy rd%d, wr%d (0x%08x): "
 685                         "%d ranks, %cDIMMs\n",
 686                         i,
 687                         RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
 688                         data,
 689                         pvt->channel[i].ranks,
 690                         (data & REGISTERED_DIMM) ? 'R' : 'U');
 691
 692                 for (j = 0; j < 3; j++) {
 693                         u32 banks, ranks, rows, cols;
 694                         u32 size, npages;
 695
 696                         if (!DIMM_PRESENT(dimm_dod[j]))
 697                                 continue;
 698
 699                         banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
 700                         ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
 701                         rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
 702                         cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
 703
 704                         /* DDR3 has 8 I/O banks */
 705                         size = (rows * cols * banks * ranks) >> (20 - 3);
 706
 707                         pvt->channel[i].dimms++;
 708
 709                         debugf0("\tdimm %d %d Mb offset: %x, "
 710                                 "bank: %d, rank: %d, row: %#x, col: %#x\n",
 711                                 j, size,
 712                                 RANKOFFSET(dimm_dod[j]),
 713                                 banks, ranks, rows, cols);
 714
 715                         npages = MiB_TO_PAGES(size);
 716
 717                         csr = &mci->csrows[csrow];
 718                         csr->first_page = last_page + 1;
 719                         last_page += npages;
 720                         csr->last_page = last_page;
 721                         csr->nr_pages = npages;
 722
 723                         csr->page_mask = 0;
 724                         csr->grain = 8;
 725                         csr->csrow_idx = csrow;
 726                         csr->nr_channels = 1;
 727
 728                         csr->channels[0].chan_idx = i;
 729                         csr->channels[0].ce_count = 0;
 730
 731                         pvt->csrow_map[i][j] = csrow;
 732
 733                         switch (banks) {
 734                         case 4:
 735                                 csr->dtype = DEV_X4;
 736                                 break;
 737                         case 8:
 738                                 csr->dtype = DEV_X8;
 739                                 break;
 740                         case 16:
 741                                 csr->dtype = DEV_X16;
 742                                 break;
 743                         default:
 744                                 csr->dtype = DEV_UNKNOWN;
 745                         }
 746
 747                         csr->edac_mode = mode;
 748                         csr->mtype = mtype;
 749                         snprintf(csr->channels[0].label,
 750                                         sizeof(csr->channels[0].label),
 751                                         "CPU#%uChannel#%u_DIMM#%u",
 752                                         pvt->i7core_dev->socket, i, j);
 753
 754                         csrow++;
 755                 }
 756
 757                 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
 758                 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
 759                 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
 760                 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
 761                 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
 762                 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
 763                 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
 764                 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
 765                 debugf1("\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
 766                 for (j = 0; j < 8; j++)
 767                         debugf1("\t\t%#x\t%#x\t%#x\n",
 768                                 (value[j] >> 27) & 0x1,
 769                                 (value[j] >> 24) & 0x7,
 770                                 (value[j] & ((1 << 24) - 1)));
 771         }
 772
 773         return 0;
 774 }
 775
 776 /****************************************************************************
 777                         Error insertion routines
 778  ****************************************************************************/
 779
 780 /* The i7core has independent error injection features per channel.
 781    However, to have a simpler code, we don't allow enabling error injection
 782    on more than one channel.
 783    Also, since a change at an inject parameter will be applied only at enable,
 784    we're disabling error injection on all write calls to the sysfs nodes that
 785    controls the error code injection.
 786  */
 787 static int disable_inject(const struct mem_ctl_info *mci)
 788 {
 789         struct i7core_pvt *pvt = mci->pvt_info;
 790
 791         pvt->inject.enable = 0;
 792
 793         if (!pvt->pci_ch[pvt->inject.channel][0])
 794                 return -ENODEV;
 795
 796         pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
 797                                 MC_CHANNEL_ERROR_INJECT, 0);
 798
 799         return 0;
 800 }
 801
 802 /*
 803  * i7core inject inject.section
 804  *
 805  *      accept and store error injection inject.section value
 806  *      bit 0 - refers to the lower 32-byte half cacheline
 807  *      bit 1 - refers to the upper 32-byte half cacheline
 808  */
 809 static ssize_t i7core_inject_section_store(struct mem_ctl_info *mci,
 810                                            const char *data, size_t count)
 811 {
 812         struct i7core_pvt *pvt = mci->pvt_info;
 813         unsigned long value;
 814         int rc;
 815
 816         if (pvt->inject.enable)
 817                 disable_inject(mci);
 818
 819         rc = strict_strtoul(data, 10, &value);
 820         if ((rc < 0) || (value > 3))
 821                 return -EIO;
 822
 823         pvt->inject.section = (u32) value;
 824         return count;
 825 }
 826
 827 static ssize_t i7core_inject_section_show(struct mem_ctl_info *mci,
 828                                               char *data)
 829 {
 830         struct i7core_pvt *pvt = mci->pvt_info;
 831         return sprintf(data, "0x%08x\n", pvt->inject.section);
 832 }
 833
 834 /*
 835  * i7core inject.type
 836  *
 837  *      accept and store error injection inject.section value
 838  *      bit 0 - repeat enable - Enable error repetition
 839  *      bit 1 - inject ECC error
 840  *      bit 2 - inject parity error
 841  */
 842 static ssize_t i7core_inject_type_store(struct mem_ctl_info *mci,
 843                                         const char *data, size_t count)
 844 {
 845         struct i7core_pvt *pvt = mci->pvt_info;
 846         unsigned long value;
 847         int rc;
 848
 849         if (pvt->inject.enable)
 850                 disable_inject(mci);
 851
 852         rc = strict_strtoul(data, 10, &value);
 853         if ((rc < 0) || (value > 7))
 854                 return -EIO;
 855
 856         pvt->inject.type = (u32) value;
 857         return count;
 858 }
 859
 860 static ssize_t i7core_inject_type_show(struct mem_ctl_info *mci,
 861                                               char *data)
 862 {
 863         struct i7core_pvt *pvt = mci->pvt_info;
 864         return sprintf(data, "0x%08x\n", pvt->inject.type);
 865 }
 866
 867 /*
 868  * i7core_inject_inject.eccmask_store
 869  *
 870  * The type of error (UE/CE) will depend on the inject.eccmask value:
 871  *   Any bits set to a 1 will flip the corresponding ECC bit
 872  *   Correctable errors can be injected by flipping 1 bit or the bits within
 873  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 874  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 875  *   uncorrectable error to be injected.
 876  */
 877 static ssize_t i7core_inject_eccmask_store(struct mem_ctl_info *mci,
 878                                         const char *data, size_t count)
 879 {
 880         struct i7core_pvt *pvt = mci->pvt_info;
 881         unsigned long value;
 882         int rc;
 883
 884         if (pvt->inject.enable)
 885                 disable_inject(mci);
 886
 887         rc = strict_strtoul(data, 10, &value);
 888         if (rc < 0)
 889                 return -EIO;
 890
 891         pvt->inject.eccmask = (u32) value;
 892         return count;
 893 }
 894
 895 static ssize_t i7core_inject_eccmask_show(struct mem_ctl_info *mci,
 896                                               char *data)
 897 {
 898         struct i7core_pvt *pvt = mci->pvt_info;
 899         return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
 900 }
 901
 902 /*
 903  * i7core_addrmatch
 904  *
 905  * The type of error (UE/CE) will depend on the inject.eccmask value:
 906  *   Any bits set to a 1 will flip the corresponding ECC bit
 907  *   Correctable errors can be injected by flipping 1 bit or the bits within
 908  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 909  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 910  *   uncorrectable error to be injected.
 911  */
 912
 913 #define DECLARE_ADDR_MATCH(param, limit)                        \
 914 static ssize_t i7core_inject_store_##param(                     \
 915                 struct mem_ctl_info *mci,                       \
 916                 const char *data, size_t count)                 \
 917 {                                                               \
 918         struct i7core_pvt *pvt;                                 \
 919         long value;                                             \
 920         int rc;                                                 \
 921                                                                 \
 922         debugf1("%s()\n", __func__);                            \
 923         pvt = mci->pvt_info;                                    \
 924                                                                 \
 925         if (pvt->inject.enable)                                 \
 926                 disable_inject(mci);                            \
 927                                                                 \
 928         if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
 929                 value = -1;                                     \
 930         else {                                                  \
 931                 rc = strict_strtoul(data, 10, &value);          \
 932                 if ((rc < 0) || (value >= limit))               \
 933                         return -EIO;                            \
 934         }                                                       \
 935                                                                 \
 936         pvt->inject.param = value;                              \
 937                                                                 \
 938         return count;                                           \
 939 }                                                               \
 940                                                                 \
 941 static ssize_t i7core_inject_show_##param(                      \
 942                 struct mem_ctl_info *mci,                       \
 943                 char *data)                                     \
 944 {                                                               \
 945         struct i7core_pvt *pvt;                                 \
 946                                                                 \
 947         pvt = mci->pvt_info;                                    \
 948         debugf1("%s() pvt=%p\n", __func__, pvt);                \
 949         if (pvt->inject.param < 0)                              \
 950                 return sprintf(data, "any\n");                  \
 951         else                                                    \
 952                 return sprintf(data, "%d\n", pvt->inject.param);\
 953 }
 954
 955 #define ATTR_ADDR_MATCH(param)                                  \
 956         {                                                       \
 957                 .attr = {                                       \
 958                         .name = #param,                         \
 959                         .mode = (S_IRUGO | S_IWUSR)             \
 960                 },                                              \
 961                 .show  = i7core_inject_show_##param,            \
 962                 .store = i7core_inject_store_##param,           \
 963         }
 964
 965 DECLARE_ADDR_MATCH(channel, 3);
 966 DECLARE_ADDR_MATCH(dimm, 3);
 967 DECLARE_ADDR_MATCH(rank, 4);
 968 DECLARE_ADDR_MATCH(bank, 32);
 969 DECLARE_ADDR_MATCH(page, 0x10000);
 970 DECLARE_ADDR_MATCH(col, 0x4000);
 971
 972 static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
 973 {
 974         u32 read;
 975         int count;
 976
 977         debugf0("setting pci %02x:%02x.%x reg=%02x value=%08x\n",
 978                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 979                 where, val);
 980
 981         for (count = 0; count < 10; count++) {
 982                 if (count)
 983                         msleep(100);
 984                 pci_write_config_dword(dev, where, val);
 985                 pci_read_config_dword(dev, where, &read);
 986
 987                 if (read == val)
 988                         return 0;
 989         }
 990
 991         i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
 992                 "write=%08x. Read=%08x\n",
 993                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 994                 where, val, read);
 995
 996         return -EINVAL;
 997 }
 998
 999 /*
1000  * This routine prepares the Memory Controller for error injection.
1001  * The error will be injected when some process tries to write to the
1002  * memory that matches the given criteria.
1003  * The criteria can be set in terms of a mask where dimm, rank, bank, page
1004  * and col can be specified.
1005  * A -1 value for any of the mask items will make the MCU to ignore
1006  * that matching criteria for error injection.
1007  *
1008  * It should be noticed that the error will only happen after a write operation
1009  * on a memory that matches the condition. if REPEAT_EN is not enabled at
1010  * inject mask, then it will produce just one error. Otherwise, it will repeat
1011  * until the injectmask would be cleaned.
1012  *
1013  * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
1014  *    is reliable enough to check if the MC is using the
1015  *    three channels. However, this is not clear at the datasheet.
1016  */
1017 static ssize_t i7core_inject_enable_store(struct mem_ctl_info *mci,
1018                                        const char *data, size_t count)
1019 {
1020         struct i7core_pvt *pvt = mci->pvt_info;
1021         u32 injectmask;
1022         u64 mask = 0;
1023         int  rc;
1024         long enable;
1025
1026         if (!pvt->pci_ch[pvt->inject.channel][0])
1027                 return 0;
1028
1029         rc = strict_strtoul(data, 10, &enable);
1030         if ((rc < 0))
1031                 return 0;
1032
1033         if (enable) {
1034                 pvt->inject.enable = 1;
1035         } else {
1036                 disable_inject(mci);
1037                 return count;
1038         }
1039
1040         /* Sets pvt->inject.dimm mask */
1041         if (pvt->inject.dimm < 0)
1042                 mask |= 1LL << 41;
1043         else {
1044                 if (pvt->channel[pvt->inject.channel].dimms > 2)
1045                         mask |= (pvt->inject.dimm & 0x3LL) << 35;
1046                 else
1047                         mask |= (pvt->inject.dimm & 0x1LL) << 36;
1048         }
1049
1050         /* Sets pvt->inject.rank mask */
1051         if (pvt->inject.rank < 0)
1052                 mask |= 1LL << 40;
1053         else {
1054                 if (pvt->channel[pvt->inject.channel].dimms > 2)
1055                         mask |= (pvt->inject.rank & 0x1LL) << 34;
1056                 else
1057                         mask |= (pvt->inject.rank & 0x3LL) << 34;
1058         }
1059
1060         /* Sets pvt->inject.bank mask */
1061         if (pvt->inject.bank < 0)
1062                 mask |= 1LL << 39;
1063         else
1064                 mask |= (pvt->inject.bank & 0x15LL) << 30;
1065
1066         /* Sets pvt->inject.page mask */
1067         if (pvt->inject.page < 0)
1068                 mask |= 1LL << 38;
1069         else
1070                 mask |= (pvt->inject.page & 0xffff) << 14;
1071
1072         /* Sets pvt->inject.column mask */
1073         if (pvt->inject.col < 0)
1074                 mask |= 1LL << 37;
1075         else
1076                 mask |= (pvt->inject.col & 0x3fff);
1077
1078         /*
1079          * bit    0: REPEAT_EN
1080          * bits 1-2: MASK_HALF_CACHELINE
1081          * bit    3: INJECT_ECC
1082          * bit    4: INJECT_ADDR_PARITY
1083          */
1084
1085         injectmask = (pvt->inject.type & 1) |
1086                      (pvt->inject.section & 0x3) << 1 |
1087                      (pvt->inject.type & 0x6) << (3 - 1);
1088
1089         /* Unlock writes to registers - this register is write only */
1090         pci_write_config_dword(pvt->pci_noncore,
1091                                MC_CFG_CONTROL, 0x2);
1092
1093         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1094                                MC_CHANNEL_ADDR_MATCH, mask);
1095         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1096                                MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
1097
1098         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1099                                MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
1100
1101         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1102                                MC_CHANNEL_ERROR_INJECT, injectmask);
1103
1104         /*
1105          * This is something undocumented, based on my tests
1106          * Without writing 8 to this register, errors aren't injected. Not sure
1107          * why.
1108          */
1109         pci_write_config_dword(pvt->pci_noncore,
1110                                MC_CFG_CONTROL, 8);
1111
1112         debugf0("Error inject addr match 0x%016llx, ecc 0x%08x,"
1113                 " inject 0x%08x\n",
1114                 mask, pvt->inject.eccmask, injectmask);
1115
1116
1117         return count;
1118 }
1119
1120 static ssize_t i7core_inject_enable_show(struct mem_ctl_info *mci,
1121                                         char *data)
1122 {
1123         struct i7core_pvt *pvt = mci->pvt_info;
1124         u32 injectmask;
1125
1126         if (!pvt->pci_ch[pvt->inject.channel][0])
1127                 return 0;
1128
1129         pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1130                                MC_CHANNEL_ERROR_INJECT, &injectmask);
1131
1132         debugf0("Inject error read: 0x%018x\n", injectmask);
1133
1134         if (injectmask & 0x0c)
1135                 pvt->inject.enable = 1;
1136
1137         return sprintf(data, "%d\n", pvt->inject.enable);
1138 }
1139
1140 #define DECLARE_COUNTER(param)                                  \
1141 static ssize_t i7core_show_counter_##param(                     \
1142                 struct mem_ctl_info *mci,                       \
1143                 char *data)                                     \
1144 {                                                               \
1145         struct i7core_pvt *pvt = mci->pvt_info;                 \
1146                                                                 \
1147         debugf1("%s() \n", __func__);                           \
1148         if (!pvt->ce_count_available || (pvt->is_registered))   \
1149                 return sprintf(data, "data unavailable\n");     \
1150         return sprintf(data, "%lu\n",                           \
1151                         pvt->udimm_ce_count[param]);            \
1152 }
1153
1154 #define ATTR_COUNTER(param)                                     \
1155         {                                                       \
1156                 .attr = {                                       \
1157                         .name = __stringify(udimm##param),      \
1158                         .mode = (S_IRUGO | S_IWUSR)             \
1159                 },                                              \
1160                 .show  = i7core_show_counter_##param            \
1161         }
1162
1163 DECLARE_COUNTER(0);
1164 DECLARE_COUNTER(1);
1165 DECLARE_COUNTER(2);
1166
1167 /*
1168  * Sysfs struct
1169  */
1170
1171 static const struct mcidev_sysfs_attribute i7core_addrmatch_attrs[] = {
1172         ATTR_ADDR_MATCH(channel),
1173         ATTR_ADDR_MATCH(dimm),
1174         ATTR_ADDR_MATCH(rank),
1175         ATTR_ADDR_MATCH(bank),
1176         ATTR_ADDR_MATCH(page),
1177         ATTR_ADDR_MATCH(col),
1178         { } /* End of list */
1179 };
1180
1181 static const struct mcidev_sysfs_group i7core_inject_addrmatch = {
1182         .name  = "inject_addrmatch",
1183         .mcidev_attr = i7core_addrmatch_attrs,
1184 };
1185
1186 static const struct mcidev_sysfs_attribute i7core_udimm_counters_attrs[] = {
1187         ATTR_COUNTER(0),
1188         ATTR_COUNTER(1),
1189         ATTR_COUNTER(2),
1190         { .attr = { .name = NULL } }
1191 };
1192
1193 static const struct mcidev_sysfs_group i7core_udimm_counters = {
1194         .name  = "all_channel_counts",
1195         .mcidev_attr = i7core_udimm_counters_attrs,
1196 };
1197
1198 static const struct mcidev_sysfs_attribute i7core_sysfs_rdimm_attrs[] = {
1199         {
1200                 .attr = {
1201                         .name = "inject_section",
1202                         .mode = (S_IRUGO | S_IWUSR)
1203                 },
1204                 .show  = i7core_inject_section_show,
1205                 .store = i7core_inject_section_store,
1206         }, {
1207                 .attr = {
1208                         .name = "inject_type",
1209                         .mode = (S_IRUGO | S_IWUSR)
1210                 },
1211                 .show  = i7core_inject_type_show,
1212                 .store = i7core_inject_type_store,
1213         }, {
1214                 .attr = {
1215                         .name = "inject_eccmask",
1216                         .mode = (S_IRUGO | S_IWUSR)
1217                 },
1218                 .show  = i7core_inject_eccmask_show,
1219                 .store = i7core_inject_eccmask_store,
1220         }, {
1221                 .grp = &i7core_inject_addrmatch,
1222         }, {
1223                 .attr = {
1224                         .name = "inject_enable",
1225                         .mode = (S_IRUGO | S_IWUSR)
1226                 },
1227                 .show  = i7core_inject_enable_show,
1228                 .store = i7core_inject_enable_store,
1229         },
1230         { }     /* End of list */
1231 };
1232
1233 static const struct mcidev_sysfs_attribute i7core_sysfs_udimm_attrs[] = {
1234         {
1235                 .attr = {
1236                         .name = "inject_section",
1237                         .mode = (S_IRUGO | S_IWUSR)
1238                 },
1239                 .show  = i7core_inject_section_show,
1240                 .store = i7core_inject_section_store,
1241         }, {
1242                 .attr = {
1243                         .name = "inject_type",
1244                         .mode = (S_IRUGO | S_IWUSR)
1245                 },
1246                 .show  = i7core_inject_type_show,
1247                 .store = i7core_inject_type_store,
1248         }, {
1249                 .attr = {
1250                         .name = "inject_eccmask",
1251                         .mode = (S_IRUGO | S_IWUSR)
1252                 },
1253                 .show  = i7core_inject_eccmask_show,
1254                 .store = i7core_inject_eccmask_store,
1255         }, {
1256                 .grp = &i7core_inject_addrmatch,
1257         }, {
1258                 .attr = {
1259                         .name = "inject_enable",
1260                         .mode = (S_IRUGO | S_IWUSR)
1261                 },
1262                 .show  = i7core_inject_enable_show,
1263                 .store = i7core_inject_enable_store,
1264         }, {
1265                 .grp = &i7core_udimm_counters,
1266         },
1267         { }     /* End of list */
1268 };
1269
1270 /****************************************************************************
1271         Device initialization routines: put/get, init/exit
1272  ****************************************************************************/
1273
1274 /*
1275  *      i7core_put_all_devices  'put' all the devices that we have
1276  *                              reserved via 'get'
1277  */
1278 static void i7core_put_devices(struct i7core_dev *i7core_dev)
1279 {
1280         int i;
1281
1282         debugf0(__FILE__ ": %s()\n", __func__);
1283         for (i = 0; i < i7core_dev->n_devs; i++) {
1284                 struct pci_dev *pdev = i7core_dev->pdev[i];
1285                 if (!pdev)
1286                         continue;
1287                 debugf0("Removing dev %02x:%02x.%d\n",
1288                         pdev->bus->number,
1289                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1290                 pci_dev_put(pdev);
1291         }
1292 }
1293
1294 static void i7core_put_all_devices(void)
1295 {
1296         struct i7core_dev *i7core_dev, *tmp;
1297
1298         list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1299                 i7core_put_devices(i7core_dev);
1300                 free_i7core_dev(i7core_dev);
1301         }
1302 }
1303
1304 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1305 {
1306         struct pci_dev *pdev = NULL;
1307         int i;
1308
1309         /*
1310          * On Xeon 55xx, the Intel Quick Path Arch Generic Non-core pci buses
1311          * aren't announced by acpi. So, we need to use a legacy scan probing
1312          * to detect them
1313          */
1314         while (table && table->descr) {
1315                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1316                 if (unlikely(!pdev)) {
1317                         for (i = 0; i < MAX_SOCKET_BUSES; i++)
1318                                 pcibios_scan_specific_bus(255-i);
1319                 }
1320                 pci_dev_put(pdev);
1321                 table++;
1322         }
1323 }
1324
1325 static unsigned i7core_pci_lastbus(void)
1326 {
1327         int last_bus = 0, bus;
1328         struct pci_bus *b = NULL;
1329
1330         while ((b = pci_find_next_bus(b)) != NULL) {
1331                 bus = b->number;
1332                 debugf0("Found bus %d\n", bus);
1333                 if (bus > last_bus)
1334                         last_bus = bus;
1335         }
1336
1337         debugf0("Last bus %d\n", last_bus);
1338
1339         return last_bus;
1340 }
1341
1342 /*
1343  *      i7core_get_all_devices  Find and perform 'get' operation on the MCH's
1344  *                      device/functions we want to reference for this driver
1345  *
1346  *                      Need to 'get' device 16 func 1 and func 2
1347  */
1348 static int i7core_get_onedevice(struct pci_dev **prev,
1349                                 const struct pci_id_table *table,
1350                                 const unsigned devno,
1351                                 const unsigned last_bus)
1352 {
1353         struct i7core_dev *i7core_dev;
1354         const struct pci_id_descr *dev_descr = &table->descr[devno];
1355
1356         struct pci_dev *pdev = NULL;
1357         u8 bus = 0;
1358         u8 socket = 0;
1359
1360         pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1361                               dev_descr->dev_id, *prev);
1362
1363         /*
1364          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core regs
1365          * is at addr 8086:2c40, instead of 8086:2c41. So, we need
1366          * to probe for the alternate address in case of failure
1367          */
1368         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev) {
1369                 pci_dev_get(*prev);     /* pci_get_device will put it */
1370                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1371                                       PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev);
1372         }
1373
1374         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE &&
1375             !pdev) {
1376                 pci_dev_get(*prev);     /* pci_get_device will put it */
1377                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1378                                       PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT,
1379                                       *prev);
1380         }
1381
1382         if (!pdev) {
1383                 if (*prev) {
1384                         *prev = pdev;
1385                         return 0;
1386                 }
1387
1388                 if (dev_descr->optional)
1389                         return 0;
1390
1391                 if (devno == 0)
1392                         return -ENODEV;
1393
1394                 i7core_printk(KERN_INFO,
1395                         "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1396                         dev_descr->dev, dev_descr->func,
1397                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1398
1399                 /* End of list, leave */
1400                 return -ENODEV;
1401         }
1402         bus = pdev->bus->number;
1403
1404         socket = last_bus - bus;
1405
1406         i7core_dev = get_i7core_dev(socket);
1407         if (!i7core_dev) {
1408                 i7core_dev = alloc_i7core_dev(socket, table);
1409                 if (!i7core_dev) {
1410                         pci_dev_put(pdev);
1411                         return -ENOMEM;
1412                 }
1413         }
1414
1415         if (i7core_dev->pdev[devno]) {
1416                 i7core_printk(KERN_ERR,
1417                         "Duplicated device for "
1418                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1419                         bus, dev_descr->dev, dev_descr->func,
1420                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1421                 pci_dev_put(pdev);
1422                 return -ENODEV;
1423         }
1424
1425         i7core_dev->pdev[devno] = pdev;
1426
1427         /* Sanity check */
1428         if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1429                         PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1430                 i7core_printk(KERN_ERR,
1431                         "Device PCI ID %04x:%04x "
1432                         "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1433                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1434                         bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1435                         bus, dev_descr->dev, dev_descr->func);
1436                 return -ENODEV;
1437         }
1438
1439         /* Be sure that the device is enabled */
1440         if (unlikely(pci_enable_device(pdev) < 0)) {
1441                 i7core_printk(KERN_ERR,
1442                         "Couldn't enable "
1443                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1444                         bus, dev_descr->dev, dev_descr->func,
1445                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1446                 return -ENODEV;
1447         }
1448
1449         debugf0("Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1450                 socket, bus, dev_descr->dev,
1451                 dev_descr->func,
1452                 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1453
1454         /*
1455          * As stated on drivers/pci/search.c, the reference count for
1456          * @from is always decremented if it is not %NULL. So, as we need
1457          * to get all devices up to null, we need to do a get for the device
1458          */
1459         pci_dev_get(pdev);
1460
1461         *prev = pdev;
1462
1463         return 0;
1464 }
1465
1466 static int i7core_get_all_devices(void)
1467 {
1468         int i, rc, last_bus;
1469         struct pci_dev *pdev = NULL;
1470         const struct pci_id_table *table = pci_dev_table;
1471
1472         last_bus = i7core_pci_lastbus();
1473
1474         while (table && table->descr) {
1475                 for (i = 0; i < table->n_devs; i++) {
1476                         pdev = NULL;
1477                         do {
1478                                 rc = i7core_get_onedevice(&pdev, table, i,
1479                                                           last_bus);
1480                                 if (rc < 0) {
1481                                         if (i == 0) {
1482                                                 i = table->n_devs;
1483                                                 break;
1484                                         }
1485                                         i7core_put_all_devices();
1486                                         return -ENODEV;
1487                                 }
1488                         } while (pdev);
1489                 }
1490                 table++;
1491         }
1492
1493         return 0;
1494 }
1495
1496 static int mci_bind_devs(struct mem_ctl_info *mci,
1497                          struct i7core_dev *i7core_dev)
1498 {
1499         struct i7core_pvt *pvt = mci->pvt_info;
1500         struct pci_dev *pdev;
1501         int i, func, slot;
1502         char *family;
1503
1504         pvt->is_registered = false;
1505         pvt->enable_scrub  = false;
1506         for (i = 0; i < i7core_dev->n_devs; i++) {
1507                 pdev = i7core_dev->pdev[i];
1508                 if (!pdev)
1509                         continue;
1510
1511                 func = PCI_FUNC(pdev->devfn);
1512                 slot = PCI_SLOT(pdev->devfn);
1513                 if (slot == 3) {
1514                         if (unlikely(func > MAX_MCR_FUNC))
1515                                 goto error;
1516                         pvt->pci_mcr[func] = pdev;
1517                 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1518                         if (unlikely(func > MAX_CHAN_FUNC))
1519                                 goto error;
1520                         pvt->pci_ch[slot - 4][func] = pdev;
1521                 } else if (!slot && !func) {
1522                         pvt->pci_noncore = pdev;
1523
1524                         /* Detect the processor family */
1525                         switch (pdev->device) {
1526                         case PCI_DEVICE_ID_INTEL_I7_NONCORE:
1527                                 family = "Xeon 35xx/ i7core";
1528                                 pvt->enable_scrub = false;
1529                                 break;
1530                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT:
1531                                 family = "i7-800/i5-700";
1532                                 pvt->enable_scrub = false;
1533                                 break;
1534                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE:
1535                                 family = "Xeon 34xx";
1536                                 pvt->enable_scrub = false;
1537                                 break;
1538                         case PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT:
1539                                 family = "Xeon 55xx";
1540                                 pvt->enable_scrub = true;
1541                                 break;
1542                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2:
1543                                 family = "Xeon 56xx / i7-900";
1544                                 pvt->enable_scrub = true;
1545                                 break;
1546                         default:
1547                                 family = "unknown";
1548                                 pvt->enable_scrub = false;
1549                         }
1550                         debugf0("Detected a processor type %s\n", family);
1551                 } else
1552                         goto error;
1553
1554                 debugf0("Associated fn %d.%d, dev = %p, socket %d\n",
1555                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1556                         pdev, i7core_dev->socket);
1557
1558                 if (PCI_SLOT(pdev->devfn) == 3 &&
1559                         PCI_FUNC(pdev->devfn) == 2)
1560                         pvt->is_registered = true;
1561         }
1562
1563         return 0;
1564
1565 error:
1566         i7core_printk(KERN_ERR, "Device %d, function %d "
1567                       "is out of the expected range\n",
1568                       slot, func);
1569         return -EINVAL;
1570 }
1571
1572 /****************************************************************************
1573                         Error check routines
1574  ****************************************************************************/
1575 static void i7core_rdimm_update_csrow(struct mem_ctl_info *mci,
1576                                       const int chan,
1577                                       const int dimm,
1578                                       const int add)
1579 {
1580         char *msg;
1581         struct i7core_pvt *pvt = mci->pvt_info;
1582         int row = pvt->csrow_map[chan][dimm], i;
1583
1584         for (i = 0; i < add; i++) {
1585                 msg = kasprintf(GFP_KERNEL, "Corrected error "
1586                                 "(Socket=%d channel=%d dimm=%d)",
1587                                 pvt->i7core_dev->socket, chan, dimm);
1588
1589                 edac_mc_handle_fbd_ce(mci, row, 0, msg);
1590                 kfree (msg);
1591         }
1592 }
1593
1594 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1595                                          const int chan,
1596                                          const int new0,
1597                                          const int new1,
1598                                          const int new2)
1599 {
1600         struct i7core_pvt *pvt = mci->pvt_info;
1601         int add0 = 0, add1 = 0, add2 = 0;
1602         /* Updates CE counters if it is not the first time here */
1603         if (pvt->ce_count_available) {
1604                 /* Updates CE counters */
1605
1606                 add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1607                 add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1608                 add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1609
1610                 if (add2 < 0)
1611                         add2 += 0x7fff;
1612                 pvt->rdimm_ce_count[chan][2] += add2;
1613
1614                 if (add1 < 0)
1615                         add1 += 0x7fff;
1616                 pvt->rdimm_ce_count[chan][1] += add1;
1617
1618                 if (add0 < 0)
1619                         add0 += 0x7fff;
1620                 pvt->rdimm_ce_count[chan][0] += add0;
1621         } else
1622                 pvt->ce_count_available = 1;
1623
1624         /* Store the new values */
1625         pvt->rdimm_last_ce_count[chan][2] = new2;
1626         pvt->rdimm_last_ce_count[chan][1] = new1;
1627         pvt->rdimm_last_ce_count[chan][0] = new0;
1628
1629         /*updated the edac core */
1630         if (add0 != 0)
1631                 i7core_rdimm_update_csrow(mci, chan, 0, add0);
1632         if (add1 != 0)
1633                 i7core_rdimm_update_csrow(mci, chan, 1, add1);
1634         if (add2 != 0)
1635                 i7core_rdimm_update_csrow(mci, chan, 2, add2);
1636
1637 }
1638
1639 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1640 {
1641         struct i7core_pvt *pvt = mci->pvt_info;
1642         u32 rcv[3][2];
1643         int i, new0, new1, new2;
1644
1645         /*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1646         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1647                                                                 &rcv[0][0]);
1648         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1649                                                                 &rcv[0][1]);
1650         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1651                                                                 &rcv[1][0]);
1652         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1653                                                                 &rcv[1][1]);
1654         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1655                                                                 &rcv[2][0]);
1656         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1657                                                                 &rcv[2][1]);
1658         for (i = 0 ; i < 3; i++) {
1659                 debugf3("MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1660                         (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1661                 /*if the channel has 3 dimms*/
1662                 if (pvt->channel[i].dimms > 2) {
1663                         new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1664                         new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1665                         new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1666                 } else {
1667                         new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1668                                         DIMM_BOT_COR_ERR(rcv[i][0]);
1669                         new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1670                                         DIMM_BOT_COR_ERR(rcv[i][1]);
1671                         new2 = 0;
1672                 }
1673
1674                 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1675         }
1676 }
1677
1678 /* This function is based on the device 3 function 4 registers as described on:
1679  * Intel Xeon Processor 5500 Series Datasheet Volume 2
1680  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1681  * also available at:
1682  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1683  */
1684 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1685 {
1686         struct i7core_pvt *pvt = mci->pvt_info;
1687         u32 rcv1, rcv0;
1688         int new0, new1, new2;
1689
1690         if (!pvt->pci_mcr[4]) {
1691                 debugf0("%s MCR registers not found\n", __func__);
1692                 return;
1693         }
1694
1695         /* Corrected test errors */
1696         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1697         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1698
1699         /* Store the new values */
1700         new2 = DIMM2_COR_ERR(rcv1);
1701         new1 = DIMM1_COR_ERR(rcv0);
1702         new0 = DIMM0_COR_ERR(rcv0);
1703
1704         /* Updates CE counters if it is not the first time here */
1705         if (pvt->ce_count_available) {
1706                 /* Updates CE counters */
1707                 int add0, add1, add2;
1708
1709                 add2 = new2 - pvt->udimm_last_ce_count[2];
1710                 add1 = new1 - pvt->udimm_last_ce_count[1];
1711                 add0 = new0 - pvt->udimm_last_ce_count[0];
1712
1713                 if (add2 < 0)
1714                         add2 += 0x7fff;
1715                 pvt->udimm_ce_count[2] += add2;
1716
1717                 if (add1 < 0)
1718                         add1 += 0x7fff;
1719                 pvt->udimm_ce_count[1] += add1;
1720
1721                 if (add0 < 0)
1722                         add0 += 0x7fff;
1723                 pvt->udimm_ce_count[0] += add0;
1724
1725                 if (add0 | add1 | add2)
1726                         i7core_printk(KERN_ERR, "New Corrected error(s): "
1727                                       "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1728                                       add0, add1, add2);
1729         } else
1730                 pvt->ce_count_available = 1;
1731
1732         /* Store the new values */
1733         pvt->udimm_last_ce_count[2] = new2;
1734         pvt->udimm_last_ce_count[1] = new1;
1735         pvt->udimm_last_ce_count[0] = new0;
1736 }
1737
1738 /*
1739  * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1740  * Architectures Software Developer’s Manual Volume 3B.
1741  * Nehalem are defined as family 0x06, model 0x1a
1742  *
1743  * The MCA registers used here are the following ones:
1744  *     struct mce field MCA Register
1745  *     m->status        MSR_IA32_MC8_STATUS
1746  *     m->addr          MSR_IA32_MC8_ADDR
1747  *     m->misc          MSR_IA32_MC8_MISC
1748  * In the case of Nehalem, the error information is masked at .status and .misc
1749  * fields
1750  */
1751 static void i7core_mce_output_error(struct mem_ctl_info *mci,
1752                                     const struct mce *m)
1753 {
1754         struct i7core_pvt *pvt = mci->pvt_info;
1755         char *type, *optype, *err, *msg;
1756         unsigned long error = m->status & 0x1ff0000l;
1757         u32 optypenum = (m->status >> 4) & 0x07;
1758         u32 core_err_cnt = (m->status >> 38) & 0x7fff;
1759         u32 dimm = (m->misc >> 16) & 0x3;
1760         u32 channel = (m->misc >> 18) & 0x3;
1761         u32 syndrome = m->misc >> 32;
1762         u32 errnum = find_first_bit(&error, 32);
1763         int csrow;
1764
1765         if (m->mcgstatus & 1)
1766                 type = "FATAL";
1767         else
1768                 type = "NON_FATAL";
1769
1770         switch (optypenum) {
1771         case 0:
1772                 optype = "generic undef request";
1773                 break;
1774         case 1:
1775                 optype = "read error";
1776                 break;
1777         case 2:
1778                 optype = "write error";
1779                 break;
1780         case 3:
1781                 optype = "addr/cmd error";
1782                 break;
1783         case 4:
1784                 optype = "scrubbing error";
1785                 break;
1786         default:
1787                 optype = "reserved";
1788                 break;
1789         }
1790
1791         switch (errnum) {
1792         case 16:
1793                 err = "read ECC error";
1794                 break;
1795         case 17:
1796                 err = "RAS ECC error";
1797                 break;
1798         case 18:
1799                 err = "write parity error";
1800                 break;
1801         case 19:
1802                 err = "redundacy loss";
1803                 break;
1804         case 20:
1805                 err = "reserved";
1806                 break;
1807         case 21:
1808                 err = "memory range error";
1809                 break;
1810         case 22:
1811                 err = "RTID out of range";
1812                 break;
1813         case 23:
1814                 err = "address parity error";
1815                 break;
1816         case 24:
1817                 err = "byte enable parity error";
1818                 break;
1819         default:
1820                 err = "unknown";
1821         }
1822
1823         /* FIXME: should convert addr into bank and rank information */
1824         msg = kasprintf(GFP_ATOMIC,
1825                 "%s (addr = 0x%08llx, cpu=%d, Dimm=%d, Channel=%d, "
1826                 "syndrome=0x%08x, count=%d, Err=%08llx:%08llx (%s: %s))\n",
1827                 type, (long long) m->addr, m->cpu, dimm, channel,
1828                 syndrome, core_err_cnt, (long long)m->status,
1829                 (long long)m->misc, optype, err);
1830
1831         debugf0("%s", msg);
1832
1833         csrow = pvt->csrow_map[channel][dimm];
1834
1835         /* Call the helper to output message */
1836         if (m->mcgstatus & 1)
1837                 edac_mc_handle_fbd_ue(mci, csrow, 0,
1838                                 0 /* FIXME: should be channel here */, msg);
1839         else if (!pvt->is_registered)
1840                 edac_mc_handle_fbd_ce(mci, csrow,
1841                                 0 /* FIXME: should be channel here */, msg);
1842
1843         kfree(msg);
1844 }
1845
1846 /*
1847  *      i7core_check_error      Retrieve and process errors reported by the
1848  *                              hardware. Called by the Core module.
1849  */
1850 static void i7core_check_error(struct mem_ctl_info *mci)
1851 {
1852         struct i7core_pvt *pvt = mci->pvt_info;
1853         int i;
1854         unsigned count = 0;
1855         struct mce *m;
1856
1857         /*
1858          * MCE first step: Copy all mce errors into a temporary buffer
1859          * We use a double buffering here, to reduce the risk of
1860          * losing an error.
1861          */
1862         smp_rmb();
1863         count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1864                 % MCE_LOG_LEN;
1865         if (!count)
1866                 goto check_ce_error;
1867
1868         m = pvt->mce_outentry;
1869         if (pvt->mce_in + count > MCE_LOG_LEN) {
1870                 unsigned l = MCE_LOG_LEN - pvt->mce_in;
1871
1872                 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1873                 smp_wmb();
1874                 pvt->mce_in = 0;
1875                 count -= l;
1876                 m += l;
1877         }
1878         memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1879         smp_wmb();
1880         pvt->mce_in += count;
1881
1882         smp_rmb();
1883         if (pvt->mce_overrun) {
1884                 i7core_printk(KERN_ERR, "Lost %d memory errors\n",
1885                               pvt->mce_overrun);
1886                 smp_wmb();
1887                 pvt->mce_overrun = 0;
1888         }
1889
1890         /*
1891          * MCE second step: parse errors and display
1892          */
1893         for (i = 0; i < count; i++)
1894                 i7core_mce_output_error(mci, &pvt->mce_outentry[i]);
1895
1896         /*
1897          * Now, let's increment CE error counts
1898          */
1899 check_ce_error:
1900         if (!pvt->is_registered)
1901                 i7core_udimm_check_mc_ecc_err(mci);
1902         else
1903                 i7core_rdimm_check_mc_ecc_err(mci);
1904 }
1905
1906 /*
1907  * i7core_mce_check_error       Replicates mcelog routine to get errors
1908  *                              This routine simply queues mcelog errors, and
1909  *                              return. The error itself should be handled later
1910  *                              by i7core_check_error.
1911  * WARNING: As this routine should be called at NMI time, extra care should
1912  * be taken to avoid deadlocks, and to be as fast as possible.
1913  */
1914 static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val,
1915                                   void *data)
1916 {
1917         struct mce *mce = (struct mce *)data;
1918         struct i7core_dev *i7_dev;
1919         struct mem_ctl_info *mci;
1920         struct i7core_pvt *pvt;
1921
1922         i7_dev = get_i7core_dev(mce->socketid);
1923         if (!i7_dev)
1924                 return NOTIFY_DONE;
1925
1926         mci = i7_dev->mci;
1927         pvt = mci->pvt_info;
1928
1929         /*
1930          * Just let mcelog handle it if the error is
1931          * outside the memory controller
1932          */
1933         if (((mce->status & 0xffff) >> 7) != 1)
1934                 return NOTIFY_DONE;
1935
1936         /* Bank 8 registers are the only ones that we know how to handle */
1937         if (mce->bank != 8)
1938                 return NOTIFY_DONE;
1939
1940         smp_rmb();
1941         if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1942                 smp_wmb();
1943                 pvt->mce_overrun++;
1944                 return NOTIFY_DONE;
1945         }
1946
1947         /* Copy memory error at the ringbuffer */
1948         memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1949         smp_wmb();
1950         pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1951
1952         /* Handle fatal errors immediately */
1953         if (mce->mcgstatus & 1)
1954                 i7core_check_error(mci);
1955
1956         /* Advise mcelog that the errors were handled */
1957         return NOTIFY_STOP;
1958 }
1959
1960 static struct notifier_block i7_mce_dec = {
1961         .notifier_call  = i7core_mce_check_error,
1962 };
1963
1964 struct memdev_dmi_entry {
1965         u8 type;
1966         u8 length;
1967         u16 handle;
1968         u16 phys_mem_array_handle;
1969         u16 mem_err_info_handle;
1970         u16 total_width;
1971         u16 data_width;
1972         u16 size;
1973         u8 form;
1974         u8 device_set;
1975         u8 device_locator;
1976         u8 bank_locator;
1977         u8 memory_type;
1978         u16 type_detail;
1979         u16 speed;
1980         u8 manufacturer;
1981         u8 serial_number;
1982         u8 asset_tag;
1983         u8 part_number;
1984         u8 attributes;
1985         u32 extended_size;
1986         u16 conf_mem_clk_speed;
1987 } __attribute__((__packed__));
1988
1989
1990 /*
1991  * Decode the DRAM Clock Frequency, be paranoid, make sure that all
1992  * memory devices show the same speed, and if they don't then consider
1993  * all speeds to be invalid.
1994  */
1995 static void decode_dclk(const struct dmi_header *dh, void *_dclk_freq)
1996 {
1997         int *dclk_freq = _dclk_freq;
1998         u16 dmi_mem_clk_speed;
1999
2000         if (*dclk_freq == -1)
2001                 return;
2002
2003         if (dh->type == DMI_ENTRY_MEM_DEVICE) {
2004                 struct memdev_dmi_entry *memdev_dmi_entry =
2005                         (struct memdev_dmi_entry *)dh;
2006                 unsigned long conf_mem_clk_speed_offset =
2007                         (unsigned long)&memdev_dmi_entry->conf_mem_clk_speed -
2008                         (unsigned long)&memdev_dmi_entry->type;
2009                 unsigned long speed_offset =
2010                         (unsigned long)&memdev_dmi_entry->speed -
2011                         (unsigned long)&memdev_dmi_entry->type;
2012
2013                 /* Check that a DIMM is present */
2014                 if (memdev_dmi_entry->size == 0)
2015                         return;
2016
2017                 /*
2018                  * Pick the configured speed if it's available, otherwise
2019                  * pick the DIMM speed, or we don't have a speed.
2020                  */
2021                 if (memdev_dmi_entry->length > conf_mem_clk_speed_offset) {
2022                         dmi_mem_clk_speed =
2023                                 memdev_dmi_entry->conf_mem_clk_speed;
2024                 } else if (memdev_dmi_entry->length > speed_offset) {
2025                         dmi_mem_clk_speed = memdev_dmi_entry->speed;
2026                 } else {
2027                         *dclk_freq = -1;
2028                         return;
2029                 }
2030
2031                 if (*dclk_freq == 0) {
2032                         /* First pass, speed was 0 */
2033                         if (dmi_mem_clk_speed > 0) {
2034                                 /* Set speed if a valid speed is read */
2035                                 *dclk_freq = dmi_mem_clk_speed;
2036                         } else {
2037                                 /* Otherwise we don't have a valid speed */
2038                                 *dclk_freq = -1;
2039                         }
2040                 } else if (*dclk_freq > 0 &&
2041                            *dclk_freq != dmi_mem_clk_speed) {
2042                         /*
2043                          * If we have a speed, check that all DIMMS are the same
2044                          * speed, otherwise set the speed as invalid.
2045                          */
2046                         *dclk_freq = -1;
2047                 }
2048         }
2049 }
2050
2051 /*
2052  * The default DCLK frequency is used as a fallback if we
2053  * fail to find anything reliable in the DMI. The value
2054  * is taken straight from the datasheet.
2055  */
2056 #define DEFAULT_DCLK_FREQ 800
2057
2058 static int get_dclk_freq(void)
2059 {
2060         int dclk_freq = 0;
2061
2062         dmi_walk(decode_dclk, (void *)&dclk_freq);
2063
2064         if (dclk_freq < 1)
2065                 return DEFAULT_DCLK_FREQ;
2066
2067         return dclk_freq;
2068 }
2069
2070 /*
2071  * set_sdram_scrub_rate         This routine sets byte/sec bandwidth scrub rate
2072  *                              to hardware according to SCRUBINTERVAL formula
2073  *                              found in datasheet.
2074  */
2075 static int set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 new_bw)
2076 {
2077         struct i7core_pvt *pvt = mci->pvt_info;
2078         struct pci_dev *pdev;
2079         u32 dw_scrub;
2080         u32 dw_ssr;
2081
2082         /* Get data from the MC register, function 2 */
2083         pdev = pvt->pci_mcr[2];
2084         if (!pdev)
2085                 return -ENODEV;
2086
2087         pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &dw_scrub);
2088
2089         if (new_bw == 0) {
2090                 /* Prepare to disable petrol scrub */
2091                 dw_scrub &= ~STARTSCRUB;
2092                 /* Stop the patrol scrub engine */
2093                 write_and_test(pdev, MC_SCRUB_CONTROL,
2094                                dw_scrub & ~SCRUBINTERVAL_MASK);
2095
2096                 /* Get current status of scrub rate and set bit to disable */
2097                 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2098                 dw_ssr &= ~SSR_MODE_MASK;
2099                 dw_ssr |= SSR_MODE_DISABLE;
2100         } else {
2101                 const int cache_line_size = 64;
2102                 const u32 freq_dclk_mhz = pvt->dclk_freq;
2103                 unsigned long long scrub_interval;
2104                 /*
2105                  * Translate the desired scrub rate to a register value and
2106                  * program the corresponding register value.
2107                  */
2108                 scrub_interval = (unsigned long long)freq_dclk_mhz *
2109                         cache_line_size * 1000000;
2110                 do_div(scrub_interval, new_bw);
2111
2112                 if (!scrub_interval || scrub_interval > SCRUBINTERVAL_MASK)
2113                         return -EINVAL;
2114
2115                 dw_scrub = SCRUBINTERVAL_MASK & scrub_interval;
2116
2117                 /* Start the patrol scrub engine */
2118                 pci_write_config_dword(pdev, MC_SCRUB_CONTROL,
2119                                        STARTSCRUB | dw_scrub);
2120
2121                 /* Get current status of scrub rate and set bit to enable */
2122                 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2123                 dw_ssr &= ~SSR_MODE_MASK;
2124                 dw_ssr |= SSR_MODE_ENABLE;
2125         }
2126         /* Disable or enable scrubbing */
2127         pci_write_config_dword(pdev, MC_SSRCONTROL, dw_ssr);
2128
2129         return new_bw;
2130 }
2131
2132 /*
2133  * get_sdram_scrub_rate         This routine convert current scrub rate value
2134  *                              into byte/sec bandwidth accourding to
2135  *                              SCRUBINTERVAL formula found in datasheet.
2136  */
2137 static int get_sdram_scrub_rate(struct mem_ctl_info *mci)
2138 {
2139         struct i7core_pvt *pvt = mci->pvt_info;
2140         struct pci_dev *pdev;
2141         const u32 cache_line_size = 64;
2142         const u32 freq_dclk_mhz = pvt->dclk_freq;
2143         unsigned long long scrub_rate;
2144         u32 scrubval;
2145
2146         /* Get data from the MC register, function 2 */
2147         pdev = pvt->pci_mcr[2];
2148         if (!pdev)
2149                 return -ENODEV;
2150
2151         /* Get current scrub control data */
2152         pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &scrubval);
2153
2154         /* Mask highest 8-bits to 0 */
2155         scrubval &=  SCRUBINTERVAL_MASK;
2156         if (!scrubval)
2157                 return 0;
2158
2159         /* Calculate scrub rate value into byte/sec bandwidth */
2160         scrub_rate =  (unsigned long long)freq_dclk_mhz *
2161                 1000000 * cache_line_size;
2162         do_div(scrub_rate, scrubval);
2163         return (int)scrub_rate;
2164 }
2165
2166 static void enable_sdram_scrub_setting(struct mem_ctl_info *mci)
2167 {
2168         struct i7core_pvt *pvt = mci->pvt_info;
2169         u32 pci_lock;
2170
2171         /* Unlock writes to pci registers */
2172         pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2173         pci_lock &= ~0x3;
2174         pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2175                                pci_lock | MC_CFG_UNLOCK);
2176
2177         mci->set_sdram_scrub_rate = set_sdram_scrub_rate;
2178         mci->get_sdram_scrub_rate = get_sdram_scrub_rate;
2179 }
2180
2181 static void disable_sdram_scrub_setting(struct mem_ctl_info *mci)
2182 {
2183         struct i7core_pvt *pvt = mci->pvt_info;
2184         u32 pci_lock;
2185
2186         /* Lock writes to pci registers */
2187         pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2188         pci_lock &= ~0x3;
2189         pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2190                                pci_lock | MC_CFG_LOCK);
2191 }
2192
2193 static void i7core_pci_ctl_create(struct i7core_pvt *pvt)
2194 {
2195         pvt->i7core_pci = edac_pci_create_generic_ctl(
2196                                                 &pvt->i7core_dev->pdev[0]->dev,
2197                                                 EDAC_MOD_STR);
2198         if (unlikely(!pvt->i7core_pci))
2199                 i7core_printk(KERN_WARNING,
2200                               "Unable to setup PCI error report via EDAC\n");
2201 }
2202
2203 static void i7core_pci_ctl_release(struct i7core_pvt *pvt)
2204 {
2205         if (likely(pvt->i7core_pci))
2206                 edac_pci_release_generic_ctl(pvt->i7core_pci);
2207         else
2208                 i7core_printk(KERN_ERR,
2209                                 "Couldn't find mem_ctl_info for socket %d\n",
2210                                 pvt->i7core_dev->socket);
2211         pvt->i7core_pci = NULL;
2212 }
2213
2214 static void i7core_unregister_mci(struct i7core_dev *i7core_dev)
2215 {
2216         struct mem_ctl_info *mci = i7core_dev->mci;
2217         struct i7core_pvt *pvt;
2218
2219         if (unlikely(!mci || !mci->pvt_info)) {
2220                 debugf0("MC: " __FILE__ ": %s(): dev = %p\n",
2221                         __func__, &i7core_dev->pdev[0]->dev);
2222
2223                 i7core_printk(KERN_ERR, "Couldn't find mci handler\n");
2224                 return;
2225         }
2226
2227         pvt = mci->pvt_info;
2228
2229         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2230                 __func__, mci, &i7core_dev->pdev[0]->dev);
2231
2232         /* Disable scrubrate setting */
2233         if (pvt->enable_scrub)
2234                 disable_sdram_scrub_setting(mci);
2235
2236         /* Disable EDAC polling */
2237         i7core_pci_ctl_release(pvt);
2238
2239         /* Remove MC sysfs nodes */
2240         edac_mc_del_mc(mci->dev);
2241
2242         debugf1("%s: free mci struct\n", mci->ctl_name);
2243         kfree(mci->ctl_name);
2244         edac_mc_free(mci);
2245         i7core_dev->mci = NULL;
2246 }
2247
2248 static int i7core_register_mci(struct i7core_dev *i7core_dev)
2249 {
2250         struct mem_ctl_info *mci;
2251         struct i7core_pvt *pvt;
2252         int rc, channels, csrows;
2253
2254         /* Check the number of active and not disabled channels */
2255         rc = i7core_get_active_channels(i7core_dev->socket, &channels, &csrows);
2256         if (unlikely(rc < 0))
2257                 return rc;
2258
2259         /* allocate a new MC control structure */
2260         mci = edac_mc_alloc(sizeof(*pvt), csrows, channels, i7core_dev->socket);
2261         if (unlikely(!mci))
2262                 return -ENOMEM;
2263
2264         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2265                 __func__, mci, &i7core_dev->pdev[0]->dev);
2266
2267         pvt = mci->pvt_info;
2268         memset(pvt, 0, sizeof(*pvt));
2269
2270         /* Associates i7core_dev and mci for future usage */
2271         pvt->i7core_dev = i7core_dev;
2272         i7core_dev->mci = mci;
2273
2274         /*
2275          * FIXME: how to handle RDDR3 at MCI level? It is possible to have
2276          * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
2277          * memory channels
2278          */
2279         mci->mtype_cap = MEM_FLAG_DDR3;
2280         mci->edac_ctl_cap = EDAC_FLAG_NONE;
2281         mci->edac_cap = EDAC_FLAG_NONE;
2282         mci->mod_name = "i7core_edac.c";
2283         mci->mod_ver = I7CORE_REVISION;
2284         mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
2285                                   i7core_dev->socket);
2286         mci->dev_name = pci_name(i7core_dev->pdev[0]);
2287         mci->ctl_page_to_phys = NULL;
2288
2289         /* Store pci devices at mci for faster access */
2290         rc = mci_bind_devs(mci, i7core_dev);
2291         if (unlikely(rc < 0))
2292                 goto fail0;
2293
2294         if (pvt->is_registered)
2295                 mci->mc_driver_sysfs_attributes = i7core_sysfs_rdimm_attrs;
2296         else
2297                 mci->mc_driver_sysfs_attributes = i7core_sysfs_udimm_attrs;
2298
2299         /* Get dimm basic config */
2300         get_dimm_config(mci);
2301         /* record ptr to the generic device */
2302         mci->dev = &i7core_dev->pdev[0]->dev;
2303         /* Set the function pointer to an actual operation function */
2304         mci->edac_check = i7core_check_error;
2305
2306         /* Enable scrubrate setting */
2307         if (pvt->enable_scrub)
2308                 enable_sdram_scrub_setting(mci);
2309
2310         /* add this new MC control structure to EDAC's list of MCs */
2311         if (unlikely(edac_mc_add_mc(mci))) {
2312                 debugf0("MC: " __FILE__
2313                         ": %s(): failed edac_mc_add_mc()\n", __func__);
2314                 /* FIXME: perhaps some code should go here that disables error
2315                  * reporting if we just enabled it
2316                  */
2317
2318                 rc = -EINVAL;
2319                 goto fail0;
2320         }
2321
2322         /* Default error mask is any memory */
2323         pvt->inject.channel = 0;
2324         pvt->inject.dimm = -1;
2325         pvt->inject.rank = -1;
2326         pvt->inject.bank = -1;
2327         pvt->inject.page = -1;
2328         pvt->inject.col = -1;
2329
2330         /* allocating generic PCI control info */
2331         i7core_pci_ctl_create(pvt);
2332
2333         /* DCLK for scrub rate setting */
2334         pvt->dclk_freq = get_dclk_freq();
2335
2336         return 0;
2337
2338 fail0:
2339         kfree(mci->ctl_name);
2340         edac_mc_free(mci);
2341         i7core_dev->mci = NULL;
2342         return rc;
2343 }
2344
2345 /*
2346  *      i7core_probe    Probe for ONE instance of device to see if it is
2347  *                      present.
2348  *      return:
2349  *              0 for FOUND a device
2350  *              < 0 for error code
2351  */
2352
2353 static int __devinit i7core_probe(struct pci_dev *pdev,
2354                                   const struct pci_device_id *id)
2355 {
2356         int rc, count = 0;
2357         struct i7core_dev *i7core_dev;
2358
2359         /* get the pci devices we want to reserve for our use */
2360         mutex_lock(&i7core_edac_lock);
2361
2362         /*
2363          * All memory controllers are allocated at the first pass.
2364          */
2365         if (unlikely(probed >= 1)) {
2366                 mutex_unlock(&i7core_edac_lock);
2367                 return -ENODEV;
2368         }
2369         probed++;
2370
2371         rc = i7core_get_all_devices();
2372         if (unlikely(rc < 0))
2373                 goto fail0;
2374
2375         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2376                 count++;
2377                 rc = i7core_register_mci(i7core_dev);
2378                 if (unlikely(rc < 0))
2379                         goto fail1;
2380         }
2381
2382         /*
2383          * Nehalem-EX uses a different memory controller. However, as the
2384          * memory controller is not visible on some Nehalem/Nehalem-EP, we
2385          * need to indirectly probe via a X58 PCI device. The same devices
2386          * are found on (some) Nehalem-EX. So, on those machines, the
2387          * probe routine needs to return -ENODEV, as the actual Memory
2388          * Controller registers won't be detected.
2389          */
2390         if (!count) {
2391                 rc = -ENODEV;
2392                 goto fail1;
2393         }
2394
2395         i7core_printk(KERN_INFO,
2396                       "Driver loaded, %d memory controller(s) found.\n",
2397                       count);
2398
2399         mutex_unlock(&i7core_edac_lock);
2400         return 0;
2401
2402 fail1:
2403         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2404                 i7core_unregister_mci(i7core_dev);
2405
2406         i7core_put_all_devices();
2407 fail0:
2408         mutex_unlock(&i7core_edac_lock);
2409         return rc;
2410 }
2411
2412 /*
2413  *      i7core_remove   destructor for one instance of device
2414  *
2415  */
2416 static void __devexit i7core_remove(struct pci_dev *pdev)
2417 {
2418         struct i7core_dev *i7core_dev;
2419
2420         debugf0(__FILE__ ": %s()\n", __func__);
2421
2422         /*
2423          * we have a trouble here: pdev value for removal will be wrong, since
2424          * it will point to the X58 register used to detect that the machine
2425          * is a Nehalem or upper design. However, due to the way several PCI
2426          * devices are grouped together to provide MC functionality, we need
2427          * to use a different method for releasing the devices
2428          */
2429
2430         mutex_lock(&i7core_edac_lock);
2431
2432         if (unlikely(!probed)) {
2433                 mutex_unlock(&i7core_edac_lock);
2434                 return;
2435         }
2436
2437         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2438                 i7core_unregister_mci(i7core_dev);
2439
2440         /* Release PCI resources */
2441         i7core_put_all_devices();
2442
2443         probed--;
2444
2445         mutex_unlock(&i7core_edac_lock);
2446 }
2447
2448 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2449
2450 /*
2451  *      i7core_driver   pci_driver structure for this module
2452  *
2453  */
2454 static struct pci_driver i7core_driver = {
2455         .name     = "i7core_edac",
2456         .probe    = i7core_probe,
2457         .remove   = __devexit_p(i7core_remove),
2458         .id_table = i7core_pci_tbl,
2459 };
2460
2461 /*
2462  *      i7core_init             Module entry function
2463  *                      Try to initialize this module for its devices
2464  */
2465 static int __init i7core_init(void)
2466 {
2467         int pci_rc;
2468
2469         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2470
2471         /* Ensure that the OPSTATE is set correctly for POLL or NMI */
2472         opstate_init();
2473
2474         if (use_pci_fixup)
2475                 i7core_xeon_pci_fixup(pci_dev_table);
2476
2477         pci_rc = pci_register_driver(&i7core_driver);
2478
2479         if (pci_rc >= 0) {
2480                 atomic_notifier_chain_register(&x86_mce_decoder_chain, &i7_mce_dec);
2481                 return 0;
2482         }
2483
2484         i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2485                       pci_rc);
2486
2487         return pci_rc;
2488 }
2489
2490 /*
2491  *      i7core_exit()   Module exit function
2492  *                      Unregister the driver
2493  */
2494 static void __exit i7core_exit(void)
2495 {
2496         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2497         pci_unregister_driver(&i7core_driver);
2498         atomic_notifier_chain_unregister(&x86_mce_decoder_chain, &i7_mce_dec);
2499 }
2500
2501 module_init(i7core_init);
2502 module_exit(i7core_exit);
2503
2504 MODULE_LICENSE("GPL");
2505 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
2506 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
2507 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2508                    I7CORE_REVISION);
2509
2510 module_param(edac_op_state, int, 0444);
2511 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");