drivers/edac/i7core_edac.c

   1 /* Intel i7 core/Nehalem Memory Controller kernel module
   2  *
   3  * This driver supports the memory controllers found on the Intel
   4  * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
   5  * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
   6  * and Westmere-EP.
   7  *
   8  * This file may be distributed under the terms of the
   9  * GNU General Public License version 2 only.
  10  *
  11  * Copyright (c) 2009-2010 by:
  12  *       Mauro Carvalho Chehab <mchehab@redhat.com>
  13  *
  14  * Red Hat Inc. http://www.redhat.com
  15  *
  16  * Forked and adapted from the i5400_edac driver
  17  *
  18  * Based on the following public Intel datasheets:
  19  * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
  20  * Datasheet, Volume 2:
  21  *      http://download.intel.com/design/processor/datashts/320835.pdf
  22  * Intel Xeon Processor 5500 Series Datasheet Volume 2
  23  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
  24  * also available at:
  25  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/init.h>
  30 #include <linux/pci.h>
  31 #include <linux/pci_ids.h>
  32 #include <linux/slab.h>
  33 #include <linux/delay.h>
  34 #include <linux/dmi.h>
  35 #include <linux/edac.h>
  36 #include <linux/mmzone.h>
  37 #include <linux/smp.h>
  38 #include <asm/mce.h>
  39 #include <asm/processor.h>
  40 #include <asm/div64.h>
  41
  42 #include "edac_core.h"
  43
  44 /* Static vars */
  45 static LIST_HEAD(i7core_edac_list);
  46 static DEFINE_MUTEX(i7core_edac_lock);
  47 static int probed;
  48
  49 static int use_pci_fixup;
  50 module_param(use_pci_fixup, int, 0444);
  51 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
  52 /*
  53  * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
  54  * registers start at bus 255, and are not reported by BIOS.
  55  * We currently find devices with only 2 sockets. In order to support more QPI
  56  * Quick Path Interconnect, just increment this number.
  57  */
  58 #define MAX_SOCKET_BUSES        2
  59
  60
  61 /*
  62  * Alter this version for the module when modifications are made
  63  */
  64 #define I7CORE_REVISION    " Ver: 1.0.0"
  65 #define EDAC_MOD_STR      "i7core_edac"
  66
  67 /*
  68  * Debug macros
  69  */
  70 #define i7core_printk(level, fmt, arg...)                       \
  71         edac_printk(level, "i7core", fmt, ##arg)
  72
  73 #define i7core_mc_printk(mci, level, fmt, arg...)               \
  74         edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
  75
  76 /*
  77  * i7core Memory Controller Registers
  78  */
  79
  80         /* OFFSETS for Device 0 Function 0 */
  81
  82 #define MC_CFG_CONTROL  0x90
  83   #define MC_CFG_UNLOCK         0x02
  84   #define MC_CFG_LOCK           0x00
  85
  86         /* OFFSETS for Device 3 Function 0 */
  87
  88 #define MC_CONTROL      0x48
  89 #define MC_STATUS       0x4c
  90 #define MC_MAX_DOD      0x64
  91
  92 /*
  93  * OFFSETS for Device 3 Function 4, as inicated on Xeon 5500 datasheet:
  94  * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  95  */
  96
  97 #define MC_TEST_ERR_RCV1        0x60
  98   #define DIMM2_COR_ERR(r)                      ((r) & 0x7fff)
  99
 100 #define MC_TEST_ERR_RCV0        0x64
 101   #define DIMM1_COR_ERR(r)                      (((r) >> 16) & 0x7fff)
 102   #define DIMM0_COR_ERR(r)                      ((r) & 0x7fff)
 103
 104 /* OFFSETS for Device 3 Function 2, as inicated on Xeon 5500 datasheet */
 105 #define MC_SSRCONTROL           0x48
 106   #define SSR_MODE_DISABLE      0x00
 107   #define SSR_MODE_ENABLE       0x01
 108   #define SSR_MODE_MASK         0x03
 109
 110 #define MC_SCRUB_CONTROL        0x4c
 111   #define STARTSCRUB            (1 << 24)
 112   #define SCRUBINTERVAL_MASK    0xffffff
 113
 114 #define MC_COR_ECC_CNT_0        0x80
 115 #define MC_COR_ECC_CNT_1        0x84
 116 #define MC_COR_ECC_CNT_2        0x88
 117 #define MC_COR_ECC_CNT_3        0x8c
 118 #define MC_COR_ECC_CNT_4        0x90
 119 #define MC_COR_ECC_CNT_5        0x94
 120
 121 #define DIMM_TOP_COR_ERR(r)                     (((r) >> 16) & 0x7fff)
 122 #define DIMM_BOT_COR_ERR(r)                     ((r) & 0x7fff)
 123
 124
 125         /* OFFSETS for Devices 4,5 and 6 Function 0 */
 126
 127 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
 128   #define THREE_DIMMS_PRESENT           (1 << 24)
 129   #define SINGLE_QUAD_RANK_PRESENT      (1 << 23)
 130   #define QUAD_RANK_PRESENT             (1 << 22)
 131   #define REGISTERED_DIMM               (1 << 15)
 132
 133 #define MC_CHANNEL_MAPPER       0x60
 134   #define RDLCH(r, ch)          ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
 135   #define WRLCH(r, ch)          ((((r) >> (ch * 6)) & 0x07) - 1)
 136
 137 #define MC_CHANNEL_RANK_PRESENT 0x7c
 138   #define RANK_PRESENT_MASK             0xffff
 139
 140 #define MC_CHANNEL_ADDR_MATCH   0xf0
 141 #define MC_CHANNEL_ERROR_MASK   0xf8
 142 #define MC_CHANNEL_ERROR_INJECT 0xfc
 143   #define INJECT_ADDR_PARITY    0x10
 144   #define INJECT_ECC            0x08
 145   #define MASK_CACHELINE        0x06
 146   #define MASK_FULL_CACHELINE   0x06
 147   #define MASK_MSB32_CACHELINE  0x04
 148   #define MASK_LSB32_CACHELINE  0x02
 149   #define NO_MASK_CACHELINE     0x00
 150   #define REPEAT_EN             0x01
 151
 152         /* OFFSETS for Devices 4,5 and 6 Function 1 */
 153
 154 #define MC_DOD_CH_DIMM0         0x48
 155 #define MC_DOD_CH_DIMM1         0x4c
 156 #define MC_DOD_CH_DIMM2         0x50
 157   #define RANKOFFSET_MASK       ((1 << 12) | (1 << 11) | (1 << 10))
 158   #define RANKOFFSET(x)         ((x & RANKOFFSET_MASK) >> 10)
 159   #define DIMM_PRESENT_MASK     (1 << 9)
 160   #define DIMM_PRESENT(x)       (((x) & DIMM_PRESENT_MASK) >> 9)
 161   #define MC_DOD_NUMBANK_MASK           ((1 << 8) | (1 << 7))
 162   #define MC_DOD_NUMBANK(x)             (((x) & MC_DOD_NUMBANK_MASK) >> 7)
 163   #define MC_DOD_NUMRANK_MASK           ((1 << 6) | (1 << 5))
 164   #define MC_DOD_NUMRANK(x)             (((x) & MC_DOD_NUMRANK_MASK) >> 5)
 165   #define MC_DOD_NUMROW_MASK            ((1 << 4) | (1 << 3) | (1 << 2))
 166   #define MC_DOD_NUMROW(x)              (((x) & MC_DOD_NUMROW_MASK) >> 2)
 167   #define MC_DOD_NUMCOL_MASK            3
 168   #define MC_DOD_NUMCOL(x)              ((x) & MC_DOD_NUMCOL_MASK)
 169
 170 #define MC_RANK_PRESENT         0x7c
 171
 172 #define MC_SAG_CH_0     0x80
 173 #define MC_SAG_CH_1     0x84
 174 #define MC_SAG_CH_2     0x88
 175 #define MC_SAG_CH_3     0x8c
 176 #define MC_SAG_CH_4     0x90
 177 #define MC_SAG_CH_5     0x94
 178 #define MC_SAG_CH_6     0x98
 179 #define MC_SAG_CH_7     0x9c
 180
 181 #define MC_RIR_LIMIT_CH_0       0x40
 182 #define MC_RIR_LIMIT_CH_1       0x44
 183 #define MC_RIR_LIMIT_CH_2       0x48
 184 #define MC_RIR_LIMIT_CH_3       0x4C
 185 #define MC_RIR_LIMIT_CH_4       0x50
 186 #define MC_RIR_LIMIT_CH_5       0x54
 187 #define MC_RIR_LIMIT_CH_6       0x58
 188 #define MC_RIR_LIMIT_CH_7       0x5C
 189 #define MC_RIR_LIMIT_MASK       ((1 << 10) - 1)
 190
 191 #define MC_RIR_WAY_CH           0x80
 192   #define MC_RIR_WAY_OFFSET_MASK        (((1 << 14) - 1) & ~0x7)
 193   #define MC_RIR_WAY_RANK_MASK          0x7
 194
 195 /*
 196  * i7core structs
 197  */
 198
 199 #define NUM_CHANS 3
 200 #define MAX_DIMMS 3             /* Max DIMMS per channel */
 201 #define MAX_MCR_FUNC  4
 202 #define MAX_CHAN_FUNC 3
 203
 204 struct i7core_info {
 205         u32     mc_control;
 206         u32     mc_status;
 207         u32     max_dod;
 208         u32     ch_map;
 209 };
 210
 211
 212 struct i7core_inject {
 213         int     enable;
 214
 215         u32     section;
 216         u32     type;
 217         u32     eccmask;
 218
 219         /* Error address mask */
 220         int channel, dimm, rank, bank, page, col;
 221 };
 222
 223 struct i7core_channel {
 224         u32             ranks;
 225         u32             dimms;
 226 };
 227
 228 struct pci_id_descr {
 229         int                     dev;
 230         int                     func;
 231         int                     dev_id;
 232         int                     optional;
 233 };
 234
 235 struct pci_id_table {
 236         const struct pci_id_descr       *descr;
 237         int                             n_devs;
 238 };
 239
 240 struct i7core_dev {
 241         struct list_head        list;
 242         u8                      socket;
 243         struct pci_dev          **pdev;
 244         int                     n_devs;
 245         struct mem_ctl_info     *mci;
 246 };
 247
 248 struct i7core_pvt {
 249         struct pci_dev  *pci_noncore;
 250         struct pci_dev  *pci_mcr[MAX_MCR_FUNC + 1];
 251         struct pci_dev  *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
 252
 253         struct i7core_dev *i7core_dev;
 254
 255         struct i7core_info      info;
 256         struct i7core_inject    inject;
 257         struct i7core_channel   channel[NUM_CHANS];
 258
 259         int             ce_count_available;
 260         int             csrow_map[NUM_CHANS][MAX_DIMMS];
 261
 262                         /* ECC corrected errors counts per udimm */
 263         unsigned long   udimm_ce_count[MAX_DIMMS];
 264         int             udimm_last_ce_count[MAX_DIMMS];
 265                         /* ECC corrected errors counts per rdimm */
 266         unsigned long   rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
 267         int             rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
 268
 269         bool            is_registered, enable_scrub;
 270
 271         /* Fifo double buffers */
 272         struct mce              mce_entry[MCE_LOG_LEN];
 273         struct mce              mce_outentry[MCE_LOG_LEN];
 274
 275         /* Fifo in/out counters */
 276         unsigned                mce_in, mce_out;
 277
 278         /* Count indicator to show errors not got */
 279         unsigned                mce_overrun;
 280
 281         /* DCLK Frequency used for computing scrub rate */
 282         int                     dclk_freq;
 283
 284         /* Struct to control EDAC polling */
 285         struct edac_pci_ctl_info *i7core_pci;
 286 };
 287
 288 #define PCI_DESCR(device, function, device_id)  \
 289         .dev = (device),                        \
 290         .func = (function),                     \
 291         .dev_id = (device_id)
 292
 293 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
 294                 /* Memory controller */
 295         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
 296         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
 297                         /* Exists only for RDIMM */
 298         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1  },
 299         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
 300
 301                 /* Channel 0 */
 302         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
 303         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
 304         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
 305         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
 306
 307                 /* Channel 1 */
 308         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
 309         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
 310         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
 311         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
 312
 313                 /* Channel 2 */
 314         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
 315         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
 316         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
 317         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
 318
 319                 /* Generic Non-core registers */
 320         /*
 321          * This is the PCI device on i7core and on Xeon 35xx (8086:2c41)
 322          * On Xeon 55xx, however, it has a different id (8086:2c40). So,
 323          * the probing code needs to test for the other address in case of
 324          * failure of this one
 325          */
 326         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE)  },
 327
 328 };
 329
 330 static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
 331         { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR)         },
 332         { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD)      },
 333         { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST)     },
 334
 335         { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
 336         { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
 337         { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
 338         { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC)   },
 339
 340         { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
 341         { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
 342         { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
 343         { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC)   },
 344
 345         /*
 346          * This is the PCI device has an alternate address on some
 347          * processors like Core i7 860
 348          */
 349         { PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE)     },
 350 };
 351
 352 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
 353                 /* Memory controller */
 354         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2)     },
 355         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2)  },
 356                         /* Exists only for RDIMM */
 357         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1  },
 358         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
 359
 360                 /* Channel 0 */
 361         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
 362         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
 363         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
 364         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2)   },
 365
 366                 /* Channel 1 */
 367         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
 368         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
 369         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
 370         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2)   },
 371
 372                 /* Channel 2 */
 373         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
 374         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
 375         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
 376         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2)   },
 377
 378                 /* Generic Non-core registers */
 379         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2)  },
 380
 381 };
 382
 383 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 384 static const struct pci_id_table pci_dev_table[] = {
 385         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
 386         PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
 387         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
 388         {0,}                    /* 0 terminated list. */
 389 };
 390
 391 /*
 392  *      pci_device_id   table for which devices we are looking for
 393  */
 394 static DEFINE_PCI_DEVICE_TABLE(i7core_pci_tbl) = {
 395         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
 396         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
 397         {0,}                    /* 0 terminated list. */
 398 };
 399
 400 /****************************************************************************
 401                         Anciliary status routines
 402  ****************************************************************************/
 403
 404         /* MC_CONTROL bits */
 405 #define CH_ACTIVE(pvt, ch)      ((pvt)->info.mc_control & (1 << (8 + ch)))
 406 #define ECCx8(pvt)              ((pvt)->info.mc_control & (1 << 1))
 407
 408         /* MC_STATUS bits */
 409 #define ECC_ENABLED(pvt)        ((pvt)->info.mc_status & (1 << 4))
 410 #define CH_DISABLED(pvt, ch)    ((pvt)->info.mc_status & (1 << ch))
 411
 412         /* MC_MAX_DOD read functions */
 413 static inline int numdimms(u32 dimms)
 414 {
 415         return (dimms & 0x3) + 1;
 416 }
 417
 418 static inline int numrank(u32 rank)
 419 {
 420         static int ranks[4] = { 1, 2, 4, -EINVAL };
 421
 422         return ranks[rank & 0x3];
 423 }
 424
 425 static inline int numbank(u32 bank)
 426 {
 427         static int banks[4] = { 4, 8, 16, -EINVAL };
 428
 429         return banks[bank & 0x3];
 430 }
 431
 432 static inline int numrow(u32 row)
 433 {
 434         static int rows[8] = {
 435                 1 << 12, 1 << 13, 1 << 14, 1 << 15,
 436                 1 << 16, -EINVAL, -EINVAL, -EINVAL,
 437         };
 438
 439         return rows[row & 0x7];
 440 }
 441
 442 static inline int numcol(u32 col)
 443 {
 444         static int cols[8] = {
 445                 1 << 10, 1 << 11, 1 << 12, -EINVAL,
 446         };
 447         return cols[col & 0x3];
 448 }
 449
 450 static struct i7core_dev *get_i7core_dev(u8 socket)
 451 {
 452         struct i7core_dev *i7core_dev;
 453
 454         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
 455                 if (i7core_dev->socket == socket)
 456                         return i7core_dev;
 457         }
 458
 459         return NULL;
 460 }
 461
 462 static struct i7core_dev *alloc_i7core_dev(u8 socket,
 463                                            const struct pci_id_table *table)
 464 {
 465         struct i7core_dev *i7core_dev;
 466
 467         i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
 468         if (!i7core_dev)
 469                 return NULL;
 470
 471         i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev) * table->n_devs,
 472                                    GFP_KERNEL);
 473         if (!i7core_dev->pdev) {
 474                 kfree(i7core_dev);
 475                 return NULL;
 476         }
 477
 478         i7core_dev->socket = socket;
 479         i7core_dev->n_devs = table->n_devs;
 480         list_add_tail(&i7core_dev->list, &i7core_edac_list);
 481
 482         return i7core_dev;
 483 }
 484
 485 static void free_i7core_dev(struct i7core_dev *i7core_dev)
 486 {
 487         list_del(&i7core_dev->list);
 488         kfree(i7core_dev->pdev);
 489         kfree(i7core_dev);
 490 }
 491
 492 /****************************************************************************
 493                         Memory check routines
 494  ****************************************************************************/
 495 static struct pci_dev *get_pdev_slot_func(u8 socket, unsigned slot,
 496                                           unsigned func)
 497 {
 498         struct i7core_dev *i7core_dev = get_i7core_dev(socket);
 499         int i;
 500
 501         if (!i7core_dev)
 502                 return NULL;
 503
 504         for (i = 0; i < i7core_dev->n_devs; i++) {
 505                 if (!i7core_dev->pdev[i])
 506                         continue;
 507
 508                 if (PCI_SLOT(i7core_dev->pdev[i]->devfn) == slot &&
 509                     PCI_FUNC(i7core_dev->pdev[i]->devfn) == func) {
 510                         return i7core_dev->pdev[i];
 511                 }
 512         }
 513
 514         return NULL;
 515 }
 516
 517 /**
 518  * i7core_get_active_channels() - gets the number of channels and csrows
 519  * @socket:     Quick Path Interconnect socket
 520  * @channels:   Number of channels that will be returned
 521  * @csrows:     Number of csrows found
 522  *
 523  * Since EDAC core needs to know in advance the number of available channels
 524  * and csrows, in order to allocate memory for csrows/channels, it is needed
 525  * to run two similar steps. At the first step, implemented on this function,
 526  * it checks the number of csrows/channels present at one socket.
 527  * this is used in order to properly allocate the size of mci components.
 528  *
 529  * It should be noticed that none of the current available datasheets explain
 530  * or even mention how csrows are seen by the memory controller. So, we need
 531  * to add a fake description for csrows.
 532  * So, this driver is attributing one DIMM memory for one csrow.
 533  */
 534 static int i7core_get_active_channels(const u8 socket, unsigned *channels,
 535                                       unsigned *csrows)
 536 {
 537         struct pci_dev *pdev = NULL;
 538         int i, j;
 539         u32 status, control;
 540
 541         *channels = 0;
 542         *csrows = 0;
 543
 544         pdev = get_pdev_slot_func(socket, 3, 0);
 545         if (!pdev) {
 546                 i7core_printk(KERN_ERR, "Couldn't find socket %d fn 3.0!!!\n",
 547                               socket);
 548                 return -ENODEV;
 549         }
 550
 551         /* Device 3 function 0 reads */
 552         pci_read_config_dword(pdev, MC_STATUS, &status);
 553         pci_read_config_dword(pdev, MC_CONTROL, &control);
 554
 555         for (i = 0; i < NUM_CHANS; i++) {
 556                 u32 dimm_dod[3];
 557                 /* Check if the channel is active */
 558                 if (!(control & (1 << (8 + i))))
 559                         continue;
 560
 561                 /* Check if the channel is disabled */
 562                 if (status & (1 << i))
 563                         continue;
 564
 565                 pdev = get_pdev_slot_func(socket, i + 4, 1);
 566                 if (!pdev) {
 567                         i7core_printk(KERN_ERR, "Couldn't find socket %d "
 568                                                 "fn %d.%d!!!\n",
 569                                                 socket, i + 4, 1);
 570                         return -ENODEV;
 571                 }
 572                 /* Devices 4-6 function 1 */
 573                 pci_read_config_dword(pdev,
 574                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 575                 pci_read_config_dword(pdev,
 576                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 577                 pci_read_config_dword(pdev,
 578                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 579
 580                 (*channels)++;
 581
 582                 for (j = 0; j < 3; j++) {
 583                         if (!DIMM_PRESENT(dimm_dod[j]))
 584                                 continue;
 585                         (*csrows)++;
 586                 }
 587         }
 588
 589         debugf0("Number of active channels on socket %d: %d\n",
 590                 socket, *channels);
 591
 592         return 0;
 593 }
 594
 595 static int get_dimm_config(struct mem_ctl_info *mci)
 596 {
 597         struct i7core_pvt *pvt = mci->pvt_info;
 598         struct csrow_info *csr;
 599         struct pci_dev *pdev;
 600         int i, j;
 601         int csrow = 0;
 602         unsigned long last_page = 0;
 603         enum edac_type mode;
 604         enum mem_type mtype;
 605         struct dimm_info *dimm;
 606
 607         /* Get data from the MC register, function 0 */
 608         pdev = pvt->pci_mcr[0];
 609         if (!pdev)
 610                 return -ENODEV;
 611
 612         /* Device 3 function 0 reads */
 613         pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
 614         pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
 615         pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
 616         pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
 617
 618         debugf0("QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
 619                 pvt->i7core_dev->socket, pvt->info.mc_control, pvt->info.mc_status,
 620                 pvt->info.max_dod, pvt->info.ch_map);
 621
 622         if (ECC_ENABLED(pvt)) {
 623                 debugf0("ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
 624                 if (ECCx8(pvt))
 625                         mode = EDAC_S8ECD8ED;
 626                 else
 627                         mode = EDAC_S4ECD4ED;
 628         } else {
 629                 debugf0("ECC disabled\n");
 630                 mode = EDAC_NONE;
 631         }
 632
 633         /* FIXME: need to handle the error codes */
 634         debugf0("DOD Max limits: DIMMS: %d, %d-ranked, %d-banked "
 635                 "x%x x 0x%x\n",
 636                 numdimms(pvt->info.max_dod),
 637                 numrank(pvt->info.max_dod >> 2),
 638                 numbank(pvt->info.max_dod >> 4),
 639                 numrow(pvt->info.max_dod >> 6),
 640                 numcol(pvt->info.max_dod >> 9));
 641
 642         for (i = 0; i < NUM_CHANS; i++) {
 643                 u32 data, dimm_dod[3], value[8];
 644
 645                 if (!pvt->pci_ch[i][0])
 646                         continue;
 647
 648                 if (!CH_ACTIVE(pvt, i)) {
 649                         debugf0("Channel %i is not active\n", i);
 650                         continue;
 651                 }
 652                 if (CH_DISABLED(pvt, i)) {
 653                         debugf0("Channel %i is disabled\n", i);
 654                         continue;
 655                 }
 656
 657                 /* Devices 4-6 function 0 */
 658                 pci_read_config_dword(pvt->pci_ch[i][0],
 659                                 MC_CHANNEL_DIMM_INIT_PARAMS, &data);
 660
 661                 pvt->channel[i].ranks = (data & QUAD_RANK_PRESENT) ?
 662                                                 4 : 2;
 663
 664                 if (data & REGISTERED_DIMM)
 665                         mtype = MEM_RDDR3;
 666                 else
 667                         mtype = MEM_DDR3;
 668 #if 0
 669                 if (data & THREE_DIMMS_PRESENT)
 670                         pvt->channel[i].dimms = 3;
 671                 else if (data & SINGLE_QUAD_RANK_PRESENT)
 672                         pvt->channel[i].dimms = 1;
 673                 else
 674                         pvt->channel[i].dimms = 2;
 675 #endif
 676
 677                 /* Devices 4-6 function 1 */
 678                 pci_read_config_dword(pvt->pci_ch[i][1],
 679                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 680                 pci_read_config_dword(pvt->pci_ch[i][1],
 681                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 682                 pci_read_config_dword(pvt->pci_ch[i][1],
 683                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 684
 685                 debugf0("Ch%d phy rd%d, wr%d (0x%08x): "
 686                         "%d ranks, %cDIMMs\n",
 687                         i,
 688                         RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
 689                         data,
 690                         pvt->channel[i].ranks,
 691                         (data & REGISTERED_DIMM) ? 'R' : 'U');
 692
 693                 for (j = 0; j < 3; j++) {
 694                         u32 banks, ranks, rows, cols;
 695                         u32 size, npages;
 696
 697                         if (!DIMM_PRESENT(dimm_dod[j]))
 698                                 continue;
 699
 700                         banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
 701                         ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
 702                         rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
 703                         cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
 704
 705                         /* DDR3 has 8 I/O banks */
 706                         size = (rows * cols * banks * ranks) >> (20 - 3);
 707
 708                         pvt->channel[i].dimms++;
 709
 710                         debugf0("\tdimm %d %d Mb offset: %x, "
 711                                 "bank: %d, rank: %d, row: %#x, col: %#x\n",
 712                                 j, size,
 713                                 RANKOFFSET(dimm_dod[j]),
 714                                 banks, ranks, rows, cols);
 715
 716                         npages = MiB_TO_PAGES(size);
 717
 718                         csr = &mci->csrows[csrow];
 719                         csr->first_page = last_page + 1;
 720                         last_page += npages;
 721                         csr->last_page = last_page;
 722                         csr->nr_pages = npages;
 723
 724                         csr->page_mask = 0;
 725                         csr->csrow_idx = csrow;
 726                         csr->nr_channels = 1;
 727
 728                         csr->channels[0].chan_idx = i;
 729                         csr->channels[0].ce_count = 0;
 730
 731                         pvt->csrow_map[i][j] = csrow;
 732
 733                         dimm = csr->channels[0].dimm;
 734                         switch (banks) {
 735                         case 4:
 736                                 dimm->dtype = DEV_X4;
 737                                 break;
 738                         case 8:
 739                                 dimm->dtype = DEV_X8;
 740                                 break;
 741                         case 16:
 742                                 dimm->dtype = DEV_X16;
 743                                 break;
 744                         default:
 745                                 dimm->dtype = DEV_UNKNOWN;
 746                         }
 747
 748                         snprintf(dimm->label, sizeof(dimm->label),
 749                                  "CPU#%uChannel#%u_DIMM#%u",
 750                                  pvt->i7core_dev->socket, i, j);
 751                         dimm->grain = 8;
 752                         dimm->edac_mode = mode;
 753                         dimm->mtype = mtype;
 754                 }
 755
 756                 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
 757                 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
 758                 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
 759                 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
 760                 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
 761                 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
 762                 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
 763                 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
 764                 debugf1("\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
 765                 for (j = 0; j < 8; j++)
 766                         debugf1("\t\t%#x\t%#x\t%#x\n",
 767                                 (value[j] >> 27) & 0x1,
 768                                 (value[j] >> 24) & 0x7,
 769                                 (value[j] & ((1 << 24) - 1)));
 770         }
 771
 772         return 0;
 773 }
 774
 775 /****************************************************************************
 776                         Error insertion routines
 777  ****************************************************************************/
 778
 779 /* The i7core has independent error injection features per channel.
 780    However, to have a simpler code, we don't allow enabling error injection
 781    on more than one channel.
 782    Also, since a change at an inject parameter will be applied only at enable,
 783    we're disabling error injection on all write calls to the sysfs nodes that
 784    controls the error code injection.
 785  */
 786 static int disable_inject(const struct mem_ctl_info *mci)
 787 {
 788         struct i7core_pvt *pvt = mci->pvt_info;
 789
 790         pvt->inject.enable = 0;
 791
 792         if (!pvt->pci_ch[pvt->inject.channel][0])
 793                 return -ENODEV;
 794
 795         pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
 796                                 MC_CHANNEL_ERROR_INJECT, 0);
 797
 798         return 0;
 799 }
 800
 801 /*
 802  * i7core inject inject.section
 803  *
 804  *      accept and store error injection inject.section value
 805  *      bit 0 - refers to the lower 32-byte half cacheline
 806  *      bit 1 - refers to the upper 32-byte half cacheline
 807  */
 808 static ssize_t i7core_inject_section_store(struct mem_ctl_info *mci,
 809                                            const char *data, size_t count)
 810 {
 811         struct i7core_pvt *pvt = mci->pvt_info;
 812         unsigned long value;
 813         int rc;
 814
 815         if (pvt->inject.enable)
 816                 disable_inject(mci);
 817
 818         rc = strict_strtoul(data, 10, &value);
 819         if ((rc < 0) || (value > 3))
 820                 return -EIO;
 821
 822         pvt->inject.section = (u32) value;
 823         return count;
 824 }
 825
 826 static ssize_t i7core_inject_section_show(struct mem_ctl_info *mci,
 827                                               char *data)
 828 {
 829         struct i7core_pvt *pvt = mci->pvt_info;
 830         return sprintf(data, "0x%08x\n", pvt->inject.section);
 831 }
 832
 833 /*
 834  * i7core inject.type
 835  *
 836  *      accept and store error injection inject.section value
 837  *      bit 0 - repeat enable - Enable error repetition
 838  *      bit 1 - inject ECC error
 839  *      bit 2 - inject parity error
 840  */
 841 static ssize_t i7core_inject_type_store(struct mem_ctl_info *mci,
 842                                         const char *data, size_t count)
 843 {
 844         struct i7core_pvt *pvt = mci->pvt_info;
 845         unsigned long value;
 846         int rc;
 847
 848         if (pvt->inject.enable)
 849                 disable_inject(mci);
 850
 851         rc = strict_strtoul(data, 10, &value);
 852         if ((rc < 0) || (value > 7))
 853                 return -EIO;
 854
 855         pvt->inject.type = (u32) value;
 856         return count;
 857 }
 858
 859 static ssize_t i7core_inject_type_show(struct mem_ctl_info *mci,
 860                                               char *data)
 861 {
 862         struct i7core_pvt *pvt = mci->pvt_info;
 863         return sprintf(data, "0x%08x\n", pvt->inject.type);
 864 }
 865
 866 /*
 867  * i7core_inject_inject.eccmask_store
 868  *
 869  * The type of error (UE/CE) will depend on the inject.eccmask value:
 870  *   Any bits set to a 1 will flip the corresponding ECC bit
 871  *   Correctable errors can be injected by flipping 1 bit or the bits within
 872  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 873  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 874  *   uncorrectable error to be injected.
 875  */
 876 static ssize_t i7core_inject_eccmask_store(struct mem_ctl_info *mci,
 877                                         const char *data, size_t count)
 878 {
 879         struct i7core_pvt *pvt = mci->pvt_info;
 880         unsigned long value;
 881         int rc;
 882
 883         if (pvt->inject.enable)
 884                 disable_inject(mci);
 885
 886         rc = strict_strtoul(data, 10, &value);
 887         if (rc < 0)
 888                 return -EIO;
 889
 890         pvt->inject.eccmask = (u32) value;
 891         return count;
 892 }
 893
 894 static ssize_t i7core_inject_eccmask_show(struct mem_ctl_info *mci,
 895                                               char *data)
 896 {
 897         struct i7core_pvt *pvt = mci->pvt_info;
 898         return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
 899 }
 900
 901 /*
 902  * i7core_addrmatch
 903  *
 904  * The type of error (UE/CE) will depend on the inject.eccmask value:
 905  *   Any bits set to a 1 will flip the corresponding ECC bit
 906  *   Correctable errors can be injected by flipping 1 bit or the bits within
 907  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 908  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 909  *   uncorrectable error to be injected.
 910  */
 911
 912 #define DECLARE_ADDR_MATCH(param, limit)                        \
 913 static ssize_t i7core_inject_store_##param(                     \
 914                 struct mem_ctl_info *mci,                       \
 915                 const char *data, size_t count)                 \
 916 {                                                               \
 917         struct i7core_pvt *pvt;                                 \
 918         long value;                                             \
 919         int rc;                                                 \
 920                                                                 \
 921         debugf1("%s()\n", __func__);                            \
 922         pvt = mci->pvt_info;                                    \
 923                                                                 \
 924         if (pvt->inject.enable)                                 \
 925                 disable_inject(mci);                            \
 926                                                                 \
 927         if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
 928                 value = -1;                                     \
 929         else {                                                  \
 930                 rc = strict_strtoul(data, 10, &value);          \
 931                 if ((rc < 0) || (value >= limit))               \
 932                         return -EIO;                            \
 933         }                                                       \
 934                                                                 \
 935         pvt->inject.param = value;                              \
 936                                                                 \
 937         return count;                                           \
 938 }                                                               \
 939                                                                 \
 940 static ssize_t i7core_inject_show_##param(                      \
 941                 struct mem_ctl_info *mci,                       \
 942                 char *data)                                     \
 943 {                                                               \
 944         struct i7core_pvt *pvt;                                 \
 945                                                                 \
 946         pvt = mci->pvt_info;                                    \
 947         debugf1("%s() pvt=%p\n", __func__, pvt);                \
 948         if (pvt->inject.param < 0)                              \
 949                 return sprintf(data, "any\n");                  \
 950         else                                                    \
 951                 return sprintf(data, "%d\n", pvt->inject.param);\
 952 }
 953
 954 #define ATTR_ADDR_MATCH(param)                                  \
 955         {                                                       \
 956                 .attr = {                                       \
 957                         .name = #param,                         \
 958                         .mode = (S_IRUGO | S_IWUSR)             \
 959                 },                                              \
 960                 .show  = i7core_inject_show_##param,            \
 961                 .store = i7core_inject_store_##param,           \
 962         }
 963
 964 DECLARE_ADDR_MATCH(channel, 3);
 965 DECLARE_ADDR_MATCH(dimm, 3);
 966 DECLARE_ADDR_MATCH(rank, 4);
 967 DECLARE_ADDR_MATCH(bank, 32);
 968 DECLARE_ADDR_MATCH(page, 0x10000);
 969 DECLARE_ADDR_MATCH(col, 0x4000);
 970
 971 static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
 972 {
 973         u32 read;
 974         int count;
 975
 976         debugf0("setting pci %02x:%02x.%x reg=%02x value=%08x\n",
 977                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 978                 where, val);
 979
 980         for (count = 0; count < 10; count++) {
 981                 if (count)
 982                         msleep(100);
 983                 pci_write_config_dword(dev, where, val);
 984                 pci_read_config_dword(dev, where, &read);
 985
 986                 if (read == val)
 987                         return 0;
 988         }
 989
 990         i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
 991                 "write=%08x. Read=%08x\n",
 992                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 993                 where, val, read);
 994
 995         return -EINVAL;
 996 }
 997
 998 /*
 999  * This routine prepares the Memory Controller for error injection.
1000  * The error will be injected when some process tries to write to the
1001  * memory that matches the given criteria.
1002  * The criteria can be set in terms of a mask where dimm, rank, bank, page
1003  * and col can be specified.
1004  * A -1 value for any of the mask items will make the MCU to ignore
1005  * that matching criteria for error injection.
1006  *
1007  * It should be noticed that the error will only happen after a write operation
1008  * on a memory that matches the condition. if REPEAT_EN is not enabled at
1009  * inject mask, then it will produce just one error. Otherwise, it will repeat
1010  * until the injectmask would be cleaned.
1011  *
1012  * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
1013  *    is reliable enough to check if the MC is using the
1014  *    three channels. However, this is not clear at the datasheet.
1015  */
1016 static ssize_t i7core_inject_enable_store(struct mem_ctl_info *mci,
1017                                        const char *data, size_t count)
1018 {
1019         struct i7core_pvt *pvt = mci->pvt_info;
1020         u32 injectmask;
1021         u64 mask = 0;
1022         int  rc;
1023         long enable;
1024
1025         if (!pvt->pci_ch[pvt->inject.channel][0])
1026                 return 0;
1027
1028         rc = strict_strtoul(data, 10, &enable);
1029         if ((rc < 0))
1030                 return 0;
1031
1032         if (enable) {
1033                 pvt->inject.enable = 1;
1034         } else {
1035                 disable_inject(mci);
1036                 return count;
1037         }
1038
1039         /* Sets pvt->inject.dimm mask */
1040         if (pvt->inject.dimm < 0)
1041                 mask |= 1LL << 41;
1042         else {
1043                 if (pvt->channel[pvt->inject.channel].dimms > 2)
1044                         mask |= (pvt->inject.dimm & 0x3LL) << 35;
1045                 else
1046                         mask |= (pvt->inject.dimm & 0x1LL) << 36;
1047         }
1048
1049         /* Sets pvt->inject.rank mask */
1050         if (pvt->inject.rank < 0)
1051                 mask |= 1LL << 40;
1052         else {
1053                 if (pvt->channel[pvt->inject.channel].dimms > 2)
1054                         mask |= (pvt->inject.rank & 0x1LL) << 34;
1055                 else
1056                         mask |= (pvt->inject.rank & 0x3LL) << 34;
1057         }
1058
1059         /* Sets pvt->inject.bank mask */
1060         if (pvt->inject.bank < 0)
1061                 mask |= 1LL << 39;
1062         else
1063                 mask |= (pvt->inject.bank & 0x15LL) << 30;
1064
1065         /* Sets pvt->inject.page mask */
1066         if (pvt->inject.page < 0)
1067                 mask |= 1LL << 38;
1068         else
1069                 mask |= (pvt->inject.page & 0xffff) << 14;
1070
1071         /* Sets pvt->inject.column mask */
1072         if (pvt->inject.col < 0)
1073                 mask |= 1LL << 37;
1074         else
1075                 mask |= (pvt->inject.col & 0x3fff);
1076
1077         /*
1078          * bit    0: REPEAT_EN
1079          * bits 1-2: MASK_HALF_CACHELINE
1080          * bit    3: INJECT_ECC
1081          * bit    4: INJECT_ADDR_PARITY
1082          */
1083
1084         injectmask = (pvt->inject.type & 1) |
1085                      (pvt->inject.section & 0x3) << 1 |
1086                      (pvt->inject.type & 0x6) << (3 - 1);
1087
1088         /* Unlock writes to registers - this register is write only */
1089         pci_write_config_dword(pvt->pci_noncore,
1090                                MC_CFG_CONTROL, 0x2);
1091
1092         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1093                                MC_CHANNEL_ADDR_MATCH, mask);
1094         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1095                                MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
1096
1097         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1098                                MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
1099
1100         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1101                                MC_CHANNEL_ERROR_INJECT, injectmask);
1102
1103         /*
1104          * This is something undocumented, based on my tests
1105          * Without writing 8 to this register, errors aren't injected. Not sure
1106          * why.
1107          */
1108         pci_write_config_dword(pvt->pci_noncore,
1109                                MC_CFG_CONTROL, 8);
1110
1111         debugf0("Error inject addr match 0x%016llx, ecc 0x%08x,"
1112                 " inject 0x%08x\n",
1113                 mask, pvt->inject.eccmask, injectmask);
1114
1115
1116         return count;
1117 }
1118
1119 static ssize_t i7core_inject_enable_show(struct mem_ctl_info *mci,
1120                                         char *data)
1121 {
1122         struct i7core_pvt *pvt = mci->pvt_info;
1123         u32 injectmask;
1124
1125         if (!pvt->pci_ch[pvt->inject.channel][0])
1126                 return 0;
1127
1128         pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1129                                MC_CHANNEL_ERROR_INJECT, &injectmask);
1130
1131         debugf0("Inject error read: 0x%018x\n", injectmask);
1132
1133         if (injectmask & 0x0c)
1134                 pvt->inject.enable = 1;
1135
1136         return sprintf(data, "%d\n", pvt->inject.enable);
1137 }
1138
1139 #define DECLARE_COUNTER(param)                                  \
1140 static ssize_t i7core_show_counter_##param(                     \
1141                 struct mem_ctl_info *mci,                       \
1142                 char *data)                                     \
1143 {                                                               \
1144         struct i7core_pvt *pvt = mci->pvt_info;                 \
1145                                                                 \
1146         debugf1("%s() \n", __func__);                           \
1147         if (!pvt->ce_count_available || (pvt->is_registered))   \
1148                 return sprintf(data, "data unavailable\n");     \
1149         return sprintf(data, "%lu\n",                           \
1150                         pvt->udimm_ce_count[param]);            \
1151 }
1152
1153 #define ATTR_COUNTER(param)                                     \
1154         {                                                       \
1155                 .attr = {                                       \
1156                         .name = __stringify(udimm##param),      \
1157                         .mode = (S_IRUGO | S_IWUSR)             \
1158                 },                                              \
1159                 .show  = i7core_show_counter_##param            \
1160         }
1161
1162 DECLARE_COUNTER(0);
1163 DECLARE_COUNTER(1);
1164 DECLARE_COUNTER(2);
1165
1166 /*
1167  * Sysfs struct
1168  */
1169
1170 static const struct mcidev_sysfs_attribute i7core_addrmatch_attrs[] = {
1171         ATTR_ADDR_MATCH(channel),
1172         ATTR_ADDR_MATCH(dimm),
1173         ATTR_ADDR_MATCH(rank),
1174         ATTR_ADDR_MATCH(bank),
1175         ATTR_ADDR_MATCH(page),
1176         ATTR_ADDR_MATCH(col),
1177         { } /* End of list */
1178 };
1179
1180 static const struct mcidev_sysfs_group i7core_inject_addrmatch = {
1181         .name  = "inject_addrmatch",
1182         .mcidev_attr = i7core_addrmatch_attrs,
1183 };
1184
1185 static const struct mcidev_sysfs_attribute i7core_udimm_counters_attrs[] = {
1186         ATTR_COUNTER(0),
1187         ATTR_COUNTER(1),
1188         ATTR_COUNTER(2),
1189         { .attr = { .name = NULL } }
1190 };
1191
1192 static const struct mcidev_sysfs_group i7core_udimm_counters = {
1193         .name  = "all_channel_counts",
1194         .mcidev_attr = i7core_udimm_counters_attrs,
1195 };
1196
1197 static const struct mcidev_sysfs_attribute i7core_sysfs_rdimm_attrs[] = {
1198         {
1199                 .attr = {
1200                         .name = "inject_section",
1201                         .mode = (S_IRUGO | S_IWUSR)
1202                 },
1203                 .show  = i7core_inject_section_show,
1204                 .store = i7core_inject_section_store,
1205         }, {
1206                 .attr = {
1207                         .name = "inject_type",
1208                         .mode = (S_IRUGO | S_IWUSR)
1209                 },
1210                 .show  = i7core_inject_type_show,
1211                 .store = i7core_inject_type_store,
1212         }, {
1213                 .attr = {
1214                         .name = "inject_eccmask",
1215                         .mode = (S_IRUGO | S_IWUSR)
1216                 },
1217                 .show  = i7core_inject_eccmask_show,
1218                 .store = i7core_inject_eccmask_store,
1219         }, {
1220                 .grp = &i7core_inject_addrmatch,
1221         }, {
1222                 .attr = {
1223                         .name = "inject_enable",
1224                         .mode = (S_IRUGO | S_IWUSR)
1225                 },
1226                 .show  = i7core_inject_enable_show,
1227                 .store = i7core_inject_enable_store,
1228         },
1229         { }     /* End of list */
1230 };
1231
1232 static const struct mcidev_sysfs_attribute i7core_sysfs_udimm_attrs[] = {
1233         {
1234                 .attr = {
1235                         .name = "inject_section",
1236                         .mode = (S_IRUGO | S_IWUSR)
1237                 },
1238                 .show  = i7core_inject_section_show,
1239                 .store = i7core_inject_section_store,
1240         }, {
1241                 .attr = {
1242                         .name = "inject_type",
1243                         .mode = (S_IRUGO | S_IWUSR)
1244                 },
1245                 .show  = i7core_inject_type_show,
1246                 .store = i7core_inject_type_store,
1247         }, {
1248                 .attr = {
1249                         .name = "inject_eccmask",
1250                         .mode = (S_IRUGO | S_IWUSR)
1251                 },
1252                 .show  = i7core_inject_eccmask_show,
1253                 .store = i7core_inject_eccmask_store,
1254         }, {
1255                 .grp = &i7core_inject_addrmatch,
1256         }, {
1257                 .attr = {
1258                         .name = "inject_enable",
1259                         .mode = (S_IRUGO | S_IWUSR)
1260                 },
1261                 .show  = i7core_inject_enable_show,
1262                 .store = i7core_inject_enable_store,
1263         }, {
1264                 .grp = &i7core_udimm_counters,
1265         },
1266         { }     /* End of list */
1267 };
1268
1269 /****************************************************************************
1270         Device initialization routines: put/get, init/exit
1271  ****************************************************************************/
1272
1273 /*
1274  *      i7core_put_all_devices  'put' all the devices that we have
1275  *                              reserved via 'get'
1276  */
1277 static void i7core_put_devices(struct i7core_dev *i7core_dev)
1278 {
1279         int i;
1280
1281         debugf0(__FILE__ ": %s()\n", __func__);
1282         for (i = 0; i < i7core_dev->n_devs; i++) {
1283                 struct pci_dev *pdev = i7core_dev->pdev[i];
1284                 if (!pdev)
1285                         continue;
1286                 debugf0("Removing dev %02x:%02x.%d\n",
1287                         pdev->bus->number,
1288                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1289                 pci_dev_put(pdev);
1290         }
1291 }
1292
1293 static void i7core_put_all_devices(void)
1294 {
1295         struct i7core_dev *i7core_dev, *tmp;
1296
1297         list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1298                 i7core_put_devices(i7core_dev);
1299                 free_i7core_dev(i7core_dev);
1300         }
1301 }
1302
1303 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1304 {
1305         struct pci_dev *pdev = NULL;
1306         int i;
1307
1308         /*
1309          * On Xeon 55xx, the Intel Quick Path Arch Generic Non-core pci buses
1310          * aren't announced by acpi. So, we need to use a legacy scan probing
1311          * to detect them
1312          */
1313         while (table && table->descr) {
1314                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1315                 if (unlikely(!pdev)) {
1316                         for (i = 0; i < MAX_SOCKET_BUSES; i++)
1317                                 pcibios_scan_specific_bus(255-i);
1318                 }
1319                 pci_dev_put(pdev);
1320                 table++;
1321         }
1322 }
1323
1324 static unsigned i7core_pci_lastbus(void)
1325 {
1326         int last_bus = 0, bus;
1327         struct pci_bus *b = NULL;
1328
1329         while ((b = pci_find_next_bus(b)) != NULL) {
1330                 bus = b->number;
1331                 debugf0("Found bus %d\n", bus);
1332                 if (bus > last_bus)
1333                         last_bus = bus;
1334         }
1335
1336         debugf0("Last bus %d\n", last_bus);
1337
1338         return last_bus;
1339 }
1340
1341 /*
1342  *      i7core_get_all_devices  Find and perform 'get' operation on the MCH's
1343  *                      device/functions we want to reference for this driver
1344  *
1345  *                      Need to 'get' device 16 func 1 and func 2
1346  */
1347 static int i7core_get_onedevice(struct pci_dev **prev,
1348                                 const struct pci_id_table *table,
1349                                 const unsigned devno,
1350                                 const unsigned last_bus)
1351 {
1352         struct i7core_dev *i7core_dev;
1353         const struct pci_id_descr *dev_descr = &table->descr[devno];
1354
1355         struct pci_dev *pdev = NULL;
1356         u8 bus = 0;
1357         u8 socket = 0;
1358
1359         pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1360                               dev_descr->dev_id, *prev);
1361
1362         /*
1363          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core regs
1364          * is at addr 8086:2c40, instead of 8086:2c41. So, we need
1365          * to probe for the alternate address in case of failure
1366          */
1367         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev)
1368                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1369                                       PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev);
1370
1371         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE && !pdev)
1372                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1373                                       PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT,
1374                                       *prev);
1375
1376         if (!pdev) {
1377                 if (*prev) {
1378                         *prev = pdev;
1379                         return 0;
1380                 }
1381
1382                 if (dev_descr->optional)
1383                         return 0;
1384
1385                 if (devno == 0)
1386                         return -ENODEV;
1387
1388                 i7core_printk(KERN_INFO,
1389                         "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1390                         dev_descr->dev, dev_descr->func,
1391                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1392
1393                 /* End of list, leave */
1394                 return -ENODEV;
1395         }
1396         bus = pdev->bus->number;
1397
1398         socket = last_bus - bus;
1399
1400         i7core_dev = get_i7core_dev(socket);
1401         if (!i7core_dev) {
1402                 i7core_dev = alloc_i7core_dev(socket, table);
1403                 if (!i7core_dev) {
1404                         pci_dev_put(pdev);
1405                         return -ENOMEM;
1406                 }
1407         }
1408
1409         if (i7core_dev->pdev[devno]) {
1410                 i7core_printk(KERN_ERR,
1411                         "Duplicated device for "
1412                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1413                         bus, dev_descr->dev, dev_descr->func,
1414                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1415                 pci_dev_put(pdev);
1416                 return -ENODEV;
1417         }
1418
1419         i7core_dev->pdev[devno] = pdev;
1420
1421         /* Sanity check */
1422         if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1423                         PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1424                 i7core_printk(KERN_ERR,
1425                         "Device PCI ID %04x:%04x "
1426                         "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1427                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1428                         bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1429                         bus, dev_descr->dev, dev_descr->func);
1430                 return -ENODEV;
1431         }
1432
1433         /* Be sure that the device is enabled */
1434         if (unlikely(pci_enable_device(pdev) < 0)) {
1435                 i7core_printk(KERN_ERR,
1436                         "Couldn't enable "
1437                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1438                         bus, dev_descr->dev, dev_descr->func,
1439                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1440                 return -ENODEV;
1441         }
1442
1443         debugf0("Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1444                 socket, bus, dev_descr->dev,
1445                 dev_descr->func,
1446                 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1447
1448         /*
1449          * As stated on drivers/pci/search.c, the reference count for
1450          * @from is always decremented if it is not %NULL. So, as we need
1451          * to get all devices up to null, we need to do a get for the device
1452          */
1453         pci_dev_get(pdev);
1454
1455         *prev = pdev;
1456
1457         return 0;
1458 }
1459
1460 static int i7core_get_all_devices(void)
1461 {
1462         int i, rc, last_bus;
1463         struct pci_dev *pdev = NULL;
1464         const struct pci_id_table *table = pci_dev_table;
1465
1466         last_bus = i7core_pci_lastbus();
1467
1468         while (table && table->descr) {
1469                 for (i = 0; i < table->n_devs; i++) {
1470                         pdev = NULL;
1471                         do {
1472                                 rc = i7core_get_onedevice(&pdev, table, i,
1473                                                           last_bus);
1474                                 if (rc < 0) {
1475                                         if (i == 0) {
1476                                                 i = table->n_devs;
1477                                                 break;
1478                                         }
1479                                         i7core_put_all_devices();
1480                                         return -ENODEV;
1481                                 }
1482                         } while (pdev);
1483                 }
1484                 table++;
1485         }
1486
1487         return 0;
1488 }
1489
1490 static int mci_bind_devs(struct mem_ctl_info *mci,
1491                          struct i7core_dev *i7core_dev)
1492 {
1493         struct i7core_pvt *pvt = mci->pvt_info;
1494         struct pci_dev *pdev;
1495         int i, func, slot;
1496         char *family;
1497
1498         pvt->is_registered = false;
1499         pvt->enable_scrub  = false;
1500         for (i = 0; i < i7core_dev->n_devs; i++) {
1501                 pdev = i7core_dev->pdev[i];
1502                 if (!pdev)
1503                         continue;
1504
1505                 func = PCI_FUNC(pdev->devfn);
1506                 slot = PCI_SLOT(pdev->devfn);
1507                 if (slot == 3) {
1508                         if (unlikely(func > MAX_MCR_FUNC))
1509                                 goto error;
1510                         pvt->pci_mcr[func] = pdev;
1511                 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1512                         if (unlikely(func > MAX_CHAN_FUNC))
1513                                 goto error;
1514                         pvt->pci_ch[slot - 4][func] = pdev;
1515                 } else if (!slot && !func) {
1516                         pvt->pci_noncore = pdev;
1517
1518                         /* Detect the processor family */
1519                         switch (pdev->device) {
1520                         case PCI_DEVICE_ID_INTEL_I7_NONCORE:
1521                                 family = "Xeon 35xx/ i7core";
1522                                 pvt->enable_scrub = false;
1523                                 break;
1524                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT:
1525                                 family = "i7-800/i5-700";
1526                                 pvt->enable_scrub = false;
1527                                 break;
1528                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE:
1529                                 family = "Xeon 34xx";
1530                                 pvt->enable_scrub = false;
1531                                 break;
1532                         case PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT:
1533                                 family = "Xeon 55xx";
1534                                 pvt->enable_scrub = true;
1535                                 break;
1536                         case PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2:
1537                                 family = "Xeon 56xx / i7-900";
1538                                 pvt->enable_scrub = true;
1539                                 break;
1540                         default:
1541                                 family = "unknown";
1542                                 pvt->enable_scrub = false;
1543                         }
1544                         debugf0("Detected a processor type %s\n", family);
1545                 } else
1546                         goto error;
1547
1548                 debugf0("Associated fn %d.%d, dev = %p, socket %d\n",
1549                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1550                         pdev, i7core_dev->socket);
1551
1552                 if (PCI_SLOT(pdev->devfn) == 3 &&
1553                         PCI_FUNC(pdev->devfn) == 2)
1554                         pvt->is_registered = true;
1555         }
1556
1557         return 0;
1558
1559 error:
1560         i7core_printk(KERN_ERR, "Device %d, function %d "
1561                       "is out of the expected range\n",
1562                       slot, func);
1563         return -EINVAL;
1564 }
1565
1566 /****************************************************************************
1567                         Error check routines
1568  ****************************************************************************/
1569 static void i7core_rdimm_update_csrow(struct mem_ctl_info *mci,
1570                                       const int chan,
1571                                       const int dimm,
1572                                       const int add)
1573 {
1574         char *msg;
1575         struct i7core_pvt *pvt = mci->pvt_info;
1576         int row = pvt->csrow_map[chan][dimm], i;
1577
1578         for (i = 0; i < add; i++) {
1579                 msg = kasprintf(GFP_KERNEL, "Corrected error "
1580                                 "(Socket=%d channel=%d dimm=%d)",
1581                                 pvt->i7core_dev->socket, chan, dimm);
1582
1583                 edac_mc_handle_fbd_ce(mci, row, 0, msg);
1584                 kfree (msg);
1585         }
1586 }
1587
1588 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1589                                          const int chan,
1590                                          const int new0,
1591                                          const int new1,
1592                                          const int new2)
1593 {
1594         struct i7core_pvt *pvt = mci->pvt_info;
1595         int add0 = 0, add1 = 0, add2 = 0;
1596         /* Updates CE counters if it is not the first time here */
1597         if (pvt->ce_count_available) {
1598                 /* Updates CE counters */
1599
1600                 add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1601                 add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1602                 add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1603
1604                 if (add2 < 0)
1605                         add2 += 0x7fff;
1606                 pvt->rdimm_ce_count[chan][2] += add2;
1607
1608                 if (add1 < 0)
1609                         add1 += 0x7fff;
1610                 pvt->rdimm_ce_count[chan][1] += add1;
1611
1612                 if (add0 < 0)
1613                         add0 += 0x7fff;
1614                 pvt->rdimm_ce_count[chan][0] += add0;
1615         } else
1616                 pvt->ce_count_available = 1;
1617
1618         /* Store the new values */
1619         pvt->rdimm_last_ce_count[chan][2] = new2;
1620         pvt->rdimm_last_ce_count[chan][1] = new1;
1621         pvt->rdimm_last_ce_count[chan][0] = new0;
1622
1623         /*updated the edac core */
1624         if (add0 != 0)
1625                 i7core_rdimm_update_csrow(mci, chan, 0, add0);
1626         if (add1 != 0)
1627                 i7core_rdimm_update_csrow(mci, chan, 1, add1);
1628         if (add2 != 0)
1629                 i7core_rdimm_update_csrow(mci, chan, 2, add2);
1630
1631 }
1632
1633 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1634 {
1635         struct i7core_pvt *pvt = mci->pvt_info;
1636         u32 rcv[3][2];
1637         int i, new0, new1, new2;
1638
1639         /*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1640         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1641                                                                 &rcv[0][0]);
1642         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1643                                                                 &rcv[0][1]);
1644         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1645                                                                 &rcv[1][0]);
1646         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1647                                                                 &rcv[1][1]);
1648         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1649                                                                 &rcv[2][0]);
1650         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1651                                                                 &rcv[2][1]);
1652         for (i = 0 ; i < 3; i++) {
1653                 debugf3("MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1654                         (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1655                 /*if the channel has 3 dimms*/
1656                 if (pvt->channel[i].dimms > 2) {
1657                         new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1658                         new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1659                         new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1660                 } else {
1661                         new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1662                                         DIMM_BOT_COR_ERR(rcv[i][0]);
1663                         new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1664                                         DIMM_BOT_COR_ERR(rcv[i][1]);
1665                         new2 = 0;
1666                 }
1667
1668                 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1669         }
1670 }
1671
1672 /* This function is based on the device 3 function 4 registers as described on:
1673  * Intel Xeon Processor 5500 Series Datasheet Volume 2
1674  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1675  * also available at:
1676  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1677  */
1678 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1679 {
1680         struct i7core_pvt *pvt = mci->pvt_info;
1681         u32 rcv1, rcv0;
1682         int new0, new1, new2;
1683
1684         if (!pvt->pci_mcr[4]) {
1685                 debugf0("%s MCR registers not found\n", __func__);
1686                 return;
1687         }
1688
1689         /* Corrected test errors */
1690         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1691         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1692
1693         /* Store the new values */
1694         new2 = DIMM2_COR_ERR(rcv1);
1695         new1 = DIMM1_COR_ERR(rcv0);
1696         new0 = DIMM0_COR_ERR(rcv0);
1697
1698         /* Updates CE counters if it is not the first time here */
1699         if (pvt->ce_count_available) {
1700                 /* Updates CE counters */
1701                 int add0, add1, add2;
1702
1703                 add2 = new2 - pvt->udimm_last_ce_count[2];
1704                 add1 = new1 - pvt->udimm_last_ce_count[1];
1705                 add0 = new0 - pvt->udimm_last_ce_count[0];
1706
1707                 if (add2 < 0)
1708                         add2 += 0x7fff;
1709                 pvt->udimm_ce_count[2] += add2;
1710
1711                 if (add1 < 0)
1712                         add1 += 0x7fff;
1713                 pvt->udimm_ce_count[1] += add1;
1714
1715                 if (add0 < 0)
1716                         add0 += 0x7fff;
1717                 pvt->udimm_ce_count[0] += add0;
1718
1719                 if (add0 | add1 | add2)
1720                         i7core_printk(KERN_ERR, "New Corrected error(s): "
1721                                       "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1722                                       add0, add1, add2);
1723         } else
1724                 pvt->ce_count_available = 1;
1725
1726         /* Store the new values */
1727         pvt->udimm_last_ce_count[2] = new2;
1728         pvt->udimm_last_ce_count[1] = new1;
1729         pvt->udimm_last_ce_count[0] = new0;
1730 }
1731
1732 /*
1733  * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1734  * Architectures Software Developer’s Manual Volume 3B.
1735  * Nehalem are defined as family 0x06, model 0x1a
1736  *
1737  * The MCA registers used here are the following ones:
1738  *     struct mce field MCA Register
1739  *     m->status        MSR_IA32_MC8_STATUS
1740  *     m->addr          MSR_IA32_MC8_ADDR
1741  *     m->misc          MSR_IA32_MC8_MISC
1742  * In the case of Nehalem, the error information is masked at .status and .misc
1743  * fields
1744  */
1745 static void i7core_mce_output_error(struct mem_ctl_info *mci,
1746                                     const struct mce *m)
1747 {
1748         struct i7core_pvt *pvt = mci->pvt_info;
1749         char *type, *optype, *err, *msg;
1750         unsigned long error = m->status & 0x1ff0000l;
1751         u32 optypenum = (m->status >> 4) & 0x07;
1752         u32 core_err_cnt = (m->status >> 38) & 0x7fff;
1753         u32 dimm = (m->misc >> 16) & 0x3;
1754         u32 channel = (m->misc >> 18) & 0x3;
1755         u32 syndrome = m->misc >> 32;
1756         u32 errnum = find_first_bit(&error, 32);
1757         int csrow;
1758
1759         if (m->mcgstatus & 1)
1760                 type = "FATAL";
1761         else
1762                 type = "NON_FATAL";
1763
1764         switch (optypenum) {
1765         case 0:
1766                 optype = "generic undef request";
1767                 break;
1768         case 1:
1769                 optype = "read error";
1770                 break;
1771         case 2:
1772                 optype = "write error";
1773                 break;
1774         case 3:
1775                 optype = "addr/cmd error";
1776                 break;
1777         case 4:
1778                 optype = "scrubbing error";
1779                 break;
1780         default:
1781                 optype = "reserved";
1782                 break;
1783         }
1784
1785         switch (errnum) {
1786         case 16:
1787                 err = "read ECC error";
1788                 break;
1789         case 17:
1790                 err = "RAS ECC error";
1791                 break;
1792         case 18:
1793                 err = "write parity error";
1794                 break;
1795         case 19:
1796                 err = "redundacy loss";
1797                 break;
1798         case 20:
1799                 err = "reserved";
1800                 break;
1801         case 21:
1802                 err = "memory range error";
1803                 break;
1804         case 22:
1805                 err = "RTID out of range";
1806                 break;
1807         case 23:
1808                 err = "address parity error";
1809                 break;
1810         case 24:
1811                 err = "byte enable parity error";
1812                 break;
1813         default:
1814                 err = "unknown";
1815         }
1816
1817         /* FIXME: should convert addr into bank and rank information */
1818         msg = kasprintf(GFP_ATOMIC,
1819                 "%s (addr = 0x%08llx, cpu=%d, Dimm=%d, Channel=%d, "
1820                 "syndrome=0x%08x, count=%d, Err=%08llx:%08llx (%s: %s))\n",
1821                 type, (long long) m->addr, m->cpu, dimm, channel,
1822                 syndrome, core_err_cnt, (long long)m->status,
1823                 (long long)m->misc, optype, err);
1824
1825         debugf0("%s", msg);
1826
1827         csrow = pvt->csrow_map[channel][dimm];
1828
1829         /* Call the helper to output message */
1830         if (m->mcgstatus & 1)
1831                 edac_mc_handle_fbd_ue(mci, csrow, 0,
1832                                 0 /* FIXME: should be channel here */, msg);
1833         else if (!pvt->is_registered)
1834                 edac_mc_handle_fbd_ce(mci, csrow,
1835                                 0 /* FIXME: should be channel here */, msg);
1836
1837         kfree(msg);
1838 }
1839
1840 /*
1841  *      i7core_check_error      Retrieve and process errors reported by the
1842  *                              hardware. Called by the Core module.
1843  */
1844 static void i7core_check_error(struct mem_ctl_info *mci)
1845 {
1846         struct i7core_pvt *pvt = mci->pvt_info;
1847         int i;
1848         unsigned count = 0;
1849         struct mce *m;
1850
1851         /*
1852          * MCE first step: Copy all mce errors into a temporary buffer
1853          * We use a double buffering here, to reduce the risk of
1854          * losing an error.
1855          */
1856         smp_rmb();
1857         count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1858                 % MCE_LOG_LEN;
1859         if (!count)
1860                 goto check_ce_error;
1861
1862         m = pvt->mce_outentry;
1863         if (pvt->mce_in + count > MCE_LOG_LEN) {
1864                 unsigned l = MCE_LOG_LEN - pvt->mce_in;
1865
1866                 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1867                 smp_wmb();
1868                 pvt->mce_in = 0;
1869                 count -= l;
1870                 m += l;
1871         }
1872         memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1873         smp_wmb();
1874         pvt->mce_in += count;
1875
1876         smp_rmb();
1877         if (pvt->mce_overrun) {
1878                 i7core_printk(KERN_ERR, "Lost %d memory errors\n",
1879                               pvt->mce_overrun);
1880                 smp_wmb();
1881                 pvt->mce_overrun = 0;
1882         }
1883
1884         /*
1885          * MCE second step: parse errors and display
1886          */
1887         for (i = 0; i < count; i++)
1888                 i7core_mce_output_error(mci, &pvt->mce_outentry[i]);
1889
1890         /*
1891          * Now, let's increment CE error counts
1892          */
1893 check_ce_error:
1894         if (!pvt->is_registered)
1895                 i7core_udimm_check_mc_ecc_err(mci);
1896         else
1897                 i7core_rdimm_check_mc_ecc_err(mci);
1898 }
1899
1900 /*
1901  * i7core_mce_check_error       Replicates mcelog routine to get errors
1902  *                              This routine simply queues mcelog errors, and
1903  *                              return. The error itself should be handled later
1904  *                              by i7core_check_error.
1905  * WARNING: As this routine should be called at NMI time, extra care should
1906  * be taken to avoid deadlocks, and to be as fast as possible.
1907  */
1908 static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val,
1909                                   void *data)
1910 {
1911         struct mce *mce = (struct mce *)data;
1912         struct i7core_dev *i7_dev;
1913         struct mem_ctl_info *mci;
1914         struct i7core_pvt *pvt;
1915
1916         i7_dev = get_i7core_dev(mce->socketid);
1917         if (!i7_dev)
1918                 return NOTIFY_BAD;
1919
1920         mci = i7_dev->mci;
1921         pvt = mci->pvt_info;
1922
1923         /*
1924          * Just let mcelog handle it if the error is
1925          * outside the memory controller
1926          */
1927         if (((mce->status & 0xffff) >> 7) != 1)
1928                 return NOTIFY_DONE;
1929
1930         /* Bank 8 registers are the only ones that we know how to handle */
1931         if (mce->bank != 8)
1932                 return NOTIFY_DONE;
1933
1934 #ifdef CONFIG_SMP
1935         /* Only handle if it is the right mc controller */
1936         if (mce->socketid != pvt->i7core_dev->socket)
1937                 return NOTIFY_DONE;
1938 #endif
1939
1940         smp_rmb();
1941         if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1942                 smp_wmb();
1943                 pvt->mce_overrun++;
1944                 return NOTIFY_DONE;
1945         }
1946
1947         /* Copy memory error at the ringbuffer */
1948         memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1949         smp_wmb();
1950         pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1951
1952         /* Handle fatal errors immediately */
1953         if (mce->mcgstatus & 1)
1954                 i7core_check_error(mci);
1955
1956         /* Advise mcelog that the errors were handled */
1957         return NOTIFY_STOP;
1958 }
1959
1960 static struct notifier_block i7_mce_dec = {
1961         .notifier_call  = i7core_mce_check_error,
1962 };
1963
1964 struct memdev_dmi_entry {
1965         u8 type;
1966         u8 length;
1967         u16 handle;
1968         u16 phys_mem_array_handle;
1969         u16 mem_err_info_handle;
1970         u16 total_width;
1971         u16 data_width;
1972         u16 size;
1973         u8 form;
1974         u8 device_set;
1975         u8 device_locator;
1976         u8 bank_locator;
1977         u8 memory_type;
1978         u16 type_detail;
1979         u16 speed;
1980         u8 manufacturer;
1981         u8 serial_number;
1982         u8 asset_tag;
1983         u8 part_number;
1984         u8 attributes;
1985         u32 extended_size;
1986         u16 conf_mem_clk_speed;
1987 } __attribute__((__packed__));
1988
1989
1990 /*
1991  * Decode the DRAM Clock Frequency, be paranoid, make sure that all
1992  * memory devices show the same speed, and if they don't then consider
1993  * all speeds to be invalid.
1994  */
1995 static void decode_dclk(const struct dmi_header *dh, void *_dclk_freq)
1996 {
1997         int *dclk_freq = _dclk_freq;
1998         u16 dmi_mem_clk_speed;
1999
2000         if (*dclk_freq == -1)
2001                 return;
2002
2003         if (dh->type == DMI_ENTRY_MEM_DEVICE) {
2004                 struct memdev_dmi_entry *memdev_dmi_entry =
2005                         (struct memdev_dmi_entry *)dh;
2006                 unsigned long conf_mem_clk_speed_offset =
2007                         (unsigned long)&memdev_dmi_entry->conf_mem_clk_speed -
2008                         (unsigned long)&memdev_dmi_entry->type;
2009                 unsigned long speed_offset =
2010                         (unsigned long)&memdev_dmi_entry->speed -
2011                         (unsigned long)&memdev_dmi_entry->type;
2012
2013                 /* Check that a DIMM is present */
2014                 if (memdev_dmi_entry->size == 0)
2015                         return;
2016
2017                 /*
2018                  * Pick the configured speed if it's available, otherwise
2019                  * pick the DIMM speed, or we don't have a speed.
2020                  */
2021                 if (memdev_dmi_entry->length > conf_mem_clk_speed_offset) {
2022                         dmi_mem_clk_speed =
2023                                 memdev_dmi_entry->conf_mem_clk_speed;
2024                 } else if (memdev_dmi_entry->length > speed_offset) {
2025                         dmi_mem_clk_speed = memdev_dmi_entry->speed;
2026                 } else {
2027                         *dclk_freq = -1;
2028                         return;
2029                 }
2030
2031                 if (*dclk_freq == 0) {
2032                         /* First pass, speed was 0 */
2033                         if (dmi_mem_clk_speed > 0) {
2034                                 /* Set speed if a valid speed is read */
2035                                 *dclk_freq = dmi_mem_clk_speed;
2036                         } else {
2037                                 /* Otherwise we don't have a valid speed */
2038                                 *dclk_freq = -1;
2039                         }
2040                 } else if (*dclk_freq > 0 &&
2041                            *dclk_freq != dmi_mem_clk_speed) {
2042                         /*
2043                          * If we have a speed, check that all DIMMS are the same
2044                          * speed, otherwise set the speed as invalid.
2045                          */
2046                         *dclk_freq = -1;
2047                 }
2048         }
2049 }
2050
2051 /*
2052  * The default DCLK frequency is used as a fallback if we
2053  * fail to find anything reliable in the DMI. The value
2054  * is taken straight from the datasheet.
2055  */
2056 #define DEFAULT_DCLK_FREQ 800
2057
2058 static int get_dclk_freq(void)
2059 {
2060         int dclk_freq = 0;
2061
2062         dmi_walk(decode_dclk, (void *)&dclk_freq);
2063
2064         if (dclk_freq < 1)
2065                 return DEFAULT_DCLK_FREQ;
2066
2067         return dclk_freq;
2068 }
2069
2070 /*
2071  * set_sdram_scrub_rate         This routine sets byte/sec bandwidth scrub rate
2072  *                              to hardware according to SCRUBINTERVAL formula
2073  *                              found in datasheet.
2074  */
2075 static int set_sdram_scrub_rate(struct mem_ctl_info *mci, u32 new_bw)
2076 {
2077         struct i7core_pvt *pvt = mci->pvt_info;
2078         struct pci_dev *pdev;
2079         u32 dw_scrub;
2080         u32 dw_ssr;
2081
2082         /* Get data from the MC register, function 2 */
2083         pdev = pvt->pci_mcr[2];
2084         if (!pdev)
2085                 return -ENODEV;
2086
2087         pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &dw_scrub);
2088
2089         if (new_bw == 0) {
2090                 /* Prepare to disable petrol scrub */
2091                 dw_scrub &= ~STARTSCRUB;
2092                 /* Stop the patrol scrub engine */
2093                 write_and_test(pdev, MC_SCRUB_CONTROL,
2094                                dw_scrub & ~SCRUBINTERVAL_MASK);
2095
2096                 /* Get current status of scrub rate and set bit to disable */
2097                 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2098                 dw_ssr &= ~SSR_MODE_MASK;
2099                 dw_ssr |= SSR_MODE_DISABLE;
2100         } else {
2101                 const int cache_line_size = 64;
2102                 const u32 freq_dclk_mhz = pvt->dclk_freq;
2103                 unsigned long long scrub_interval;
2104                 /*
2105                  * Translate the desired scrub rate to a register value and
2106                  * program the corresponding register value.
2107                  */
2108                 scrub_interval = (unsigned long long)freq_dclk_mhz *
2109                         cache_line_size * 1000000;
2110                 do_div(scrub_interval, new_bw);
2111
2112                 if (!scrub_interval || scrub_interval > SCRUBINTERVAL_MASK)
2113                         return -EINVAL;
2114
2115                 dw_scrub = SCRUBINTERVAL_MASK & scrub_interval;
2116
2117                 /* Start the patrol scrub engine */
2118                 pci_write_config_dword(pdev, MC_SCRUB_CONTROL,
2119                                        STARTSCRUB | dw_scrub);
2120
2121                 /* Get current status of scrub rate and set bit to enable */
2122                 pci_read_config_dword(pdev, MC_SSRCONTROL, &dw_ssr);
2123                 dw_ssr &= ~SSR_MODE_MASK;
2124                 dw_ssr |= SSR_MODE_ENABLE;
2125         }
2126         /* Disable or enable scrubbing */
2127         pci_write_config_dword(pdev, MC_SSRCONTROL, dw_ssr);
2128
2129         return new_bw;
2130 }
2131
2132 /*
2133  * get_sdram_scrub_rate         This routine convert current scrub rate value
2134  *                              into byte/sec bandwidth accourding to
2135  *                              SCRUBINTERVAL formula found in datasheet.
2136  */
2137 static int get_sdram_scrub_rate(struct mem_ctl_info *mci)
2138 {
2139         struct i7core_pvt *pvt = mci->pvt_info;
2140         struct pci_dev *pdev;
2141         const u32 cache_line_size = 64;
2142         const u32 freq_dclk_mhz = pvt->dclk_freq;
2143         unsigned long long scrub_rate;
2144         u32 scrubval;
2145
2146         /* Get data from the MC register, function 2 */
2147         pdev = pvt->pci_mcr[2];
2148         if (!pdev)
2149                 return -ENODEV;
2150
2151         /* Get current scrub control data */
2152         pci_read_config_dword(pdev, MC_SCRUB_CONTROL, &scrubval);
2153
2154         /* Mask highest 8-bits to 0 */
2155         scrubval &=  SCRUBINTERVAL_MASK;
2156         if (!scrubval)
2157                 return 0;
2158
2159         /* Calculate scrub rate value into byte/sec bandwidth */
2160         scrub_rate =  (unsigned long long)freq_dclk_mhz *
2161                 1000000 * cache_line_size;
2162         do_div(scrub_rate, scrubval);
2163         return (int)scrub_rate;
2164 }
2165
2166 static void enable_sdram_scrub_setting(struct mem_ctl_info *mci)
2167 {
2168         struct i7core_pvt *pvt = mci->pvt_info;
2169         u32 pci_lock;
2170
2171         /* Unlock writes to pci registers */
2172         pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2173         pci_lock &= ~0x3;
2174         pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2175                                pci_lock | MC_CFG_UNLOCK);
2176
2177         mci->set_sdram_scrub_rate = set_sdram_scrub_rate;
2178         mci->get_sdram_scrub_rate = get_sdram_scrub_rate;
2179 }
2180
2181 static void disable_sdram_scrub_setting(struct mem_ctl_info *mci)
2182 {
2183         struct i7core_pvt *pvt = mci->pvt_info;
2184         u32 pci_lock;
2185
2186         /* Lock writes to pci registers */
2187         pci_read_config_dword(pvt->pci_noncore, MC_CFG_CONTROL, &pci_lock);
2188         pci_lock &= ~0x3;
2189         pci_write_config_dword(pvt->pci_noncore, MC_CFG_CONTROL,
2190                                pci_lock | MC_CFG_LOCK);
2191 }
2192
2193 static void i7core_pci_ctl_create(struct i7core_pvt *pvt)
2194 {
2195         pvt->i7core_pci = edac_pci_create_generic_ctl(
2196                                                 &pvt->i7core_dev->pdev[0]->dev,
2197                                                 EDAC_MOD_STR);
2198         if (unlikely(!pvt->i7core_pci))
2199                 i7core_printk(KERN_WARNING,
2200                               "Unable to setup PCI error report via EDAC\n");
2201 }
2202
2203 static void i7core_pci_ctl_release(struct i7core_pvt *pvt)
2204 {
2205         if (likely(pvt->i7core_pci))
2206                 edac_pci_release_generic_ctl(pvt->i7core_pci);
2207         else
2208                 i7core_printk(KERN_ERR,
2209                                 "Couldn't find mem_ctl_info for socket %d\n",
2210                                 pvt->i7core_dev->socket);
2211         pvt->i7core_pci = NULL;
2212 }
2213
2214 static void i7core_unregister_mci(struct i7core_dev *i7core_dev)
2215 {
2216         struct mem_ctl_info *mci = i7core_dev->mci;
2217         struct i7core_pvt *pvt;
2218
2219         if (unlikely(!mci || !mci->pvt_info)) {
2220                 debugf0("MC: " __FILE__ ": %s(): dev = %p\n",
2221                         __func__, &i7core_dev->pdev[0]->dev);
2222
2223                 i7core_printk(KERN_ERR, "Couldn't find mci handler\n");
2224                 return;
2225         }
2226
2227         pvt = mci->pvt_info;
2228
2229         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2230                 __func__, mci, &i7core_dev->pdev[0]->dev);
2231
2232         /* Disable scrubrate setting */
2233         if (pvt->enable_scrub)
2234                 disable_sdram_scrub_setting(mci);
2235
2236         mce_unregister_decode_chain(&i7_mce_dec);
2237
2238         /* Disable EDAC polling */
2239         i7core_pci_ctl_release(pvt);
2240
2241         /* Remove MC sysfs nodes */
2242         edac_mc_del_mc(mci->dev);
2243
2244         debugf1("%s: free mci struct\n", mci->ctl_name);
2245         kfree(mci->ctl_name);
2246         edac_mc_free(mci);
2247         i7core_dev->mci = NULL;
2248 }
2249
2250 static int i7core_register_mci(struct i7core_dev *i7core_dev)
2251 {
2252         struct mem_ctl_info *mci;
2253         struct i7core_pvt *pvt;
2254         int rc, channels, csrows;
2255
2256         /* Check the number of active and not disabled channels */
2257         rc = i7core_get_active_channels(i7core_dev->socket, &channels, &csrows);
2258         if (unlikely(rc < 0))
2259                 return rc;
2260
2261         /* allocate a new MC control structure */
2262         mci = edac_mc_alloc(sizeof(*pvt), csrows, channels, i7core_dev->socket);
2263         if (unlikely(!mci))
2264                 return -ENOMEM;
2265
2266         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2267                 __func__, mci, &i7core_dev->pdev[0]->dev);
2268
2269         pvt = mci->pvt_info;
2270         memset(pvt, 0, sizeof(*pvt));
2271
2272         /* Associates i7core_dev and mci for future usage */
2273         pvt->i7core_dev = i7core_dev;
2274         i7core_dev->mci = mci;
2275
2276         /*
2277          * FIXME: how to handle RDDR3 at MCI level? It is possible to have
2278          * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
2279          * memory channels
2280          */
2281         mci->mtype_cap = MEM_FLAG_DDR3;
2282         mci->edac_ctl_cap = EDAC_FLAG_NONE;
2283         mci->edac_cap = EDAC_FLAG_NONE;
2284         mci->mod_name = "i7core_edac.c";
2285         mci->mod_ver = I7CORE_REVISION;
2286         mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
2287                                   i7core_dev->socket);
2288         mci->dev_name = pci_name(i7core_dev->pdev[0]);
2289         mci->ctl_page_to_phys = NULL;
2290
2291         /* Store pci devices at mci for faster access */
2292         rc = mci_bind_devs(mci, i7core_dev);
2293         if (unlikely(rc < 0))
2294                 goto fail0;
2295
2296         if (pvt->is_registered)
2297                 mci->mc_driver_sysfs_attributes = i7core_sysfs_rdimm_attrs;
2298         else
2299                 mci->mc_driver_sysfs_attributes = i7core_sysfs_udimm_attrs;
2300
2301         /* Get dimm basic config */
2302         get_dimm_config(mci);
2303         /* record ptr to the generic device */
2304         mci->dev = &i7core_dev->pdev[0]->dev;
2305         /* Set the function pointer to an actual operation function */
2306         mci->edac_check = i7core_check_error;
2307
2308         /* Enable scrubrate setting */
2309         if (pvt->enable_scrub)
2310                 enable_sdram_scrub_setting(mci);
2311
2312         /* add this new MC control structure to EDAC's list of MCs */
2313         if (unlikely(edac_mc_add_mc(mci))) {
2314                 debugf0("MC: " __FILE__
2315                         ": %s(): failed edac_mc_add_mc()\n", __func__);
2316                 /* FIXME: perhaps some code should go here that disables error
2317                  * reporting if we just enabled it
2318                  */
2319
2320                 rc = -EINVAL;
2321                 goto fail0;
2322         }
2323
2324         /* Default error mask is any memory */
2325         pvt->inject.channel = 0;
2326         pvt->inject.dimm = -1;
2327         pvt->inject.rank = -1;
2328         pvt->inject.bank = -1;
2329         pvt->inject.page = -1;
2330         pvt->inject.col = -1;
2331
2332         /* allocating generic PCI control info */
2333         i7core_pci_ctl_create(pvt);
2334
2335         /* DCLK for scrub rate setting */
2336         pvt->dclk_freq = get_dclk_freq();
2337
2338         mce_register_decode_chain(&i7_mce_dec);
2339
2340         return 0;
2341
2342 fail0:
2343         kfree(mci->ctl_name);
2344         edac_mc_free(mci);
2345         i7core_dev->mci = NULL;
2346         return rc;
2347 }
2348
2349 /*
2350  *      i7core_probe    Probe for ONE instance of device to see if it is
2351  *                      present.
2352  *      return:
2353  *              0 for FOUND a device
2354  *              < 0 for error code
2355  */
2356
2357 static int __devinit i7core_probe(struct pci_dev *pdev,
2358                                   const struct pci_device_id *id)
2359 {
2360         int rc, count = 0;
2361         struct i7core_dev *i7core_dev;
2362
2363         /* get the pci devices we want to reserve for our use */
2364         mutex_lock(&i7core_edac_lock);
2365
2366         /*
2367          * All memory controllers are allocated at the first pass.
2368          */
2369         if (unlikely(probed >= 1)) {
2370                 mutex_unlock(&i7core_edac_lock);
2371                 return -ENODEV;
2372         }
2373         probed++;
2374
2375         rc = i7core_get_all_devices();
2376         if (unlikely(rc < 0))
2377                 goto fail0;
2378
2379         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2380                 count++;
2381                 rc = i7core_register_mci(i7core_dev);
2382                 if (unlikely(rc < 0))
2383                         goto fail1;
2384         }
2385
2386         /*
2387          * Nehalem-EX uses a different memory controller. However, as the
2388          * memory controller is not visible on some Nehalem/Nehalem-EP, we
2389          * need to indirectly probe via a X58 PCI device. The same devices
2390          * are found on (some) Nehalem-EX. So, on those machines, the
2391          * probe routine needs to return -ENODEV, as the actual Memory
2392          * Controller registers won't be detected.
2393          */
2394         if (!count) {
2395                 rc = -ENODEV;
2396                 goto fail1;
2397         }
2398
2399         i7core_printk(KERN_INFO,
2400                       "Driver loaded, %d memory controller(s) found.\n",
2401                       count);
2402
2403         mutex_unlock(&i7core_edac_lock);
2404         return 0;
2405
2406 fail1:
2407         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2408                 i7core_unregister_mci(i7core_dev);
2409
2410         i7core_put_all_devices();
2411 fail0:
2412         mutex_unlock(&i7core_edac_lock);
2413         return rc;
2414 }
2415
2416 /*
2417  *      i7core_remove   destructor for one instance of device
2418  *
2419  */
2420 static void __devexit i7core_remove(struct pci_dev *pdev)
2421 {
2422         struct i7core_dev *i7core_dev;
2423
2424         debugf0(__FILE__ ": %s()\n", __func__);
2425
2426         /*
2427          * we have a trouble here: pdev value for removal will be wrong, since
2428          * it will point to the X58 register used to detect that the machine
2429          * is a Nehalem or upper design. However, due to the way several PCI
2430          * devices are grouped together to provide MC functionality, we need
2431          * to use a different method for releasing the devices
2432          */
2433
2434         mutex_lock(&i7core_edac_lock);
2435
2436         if (unlikely(!probed)) {
2437                 mutex_unlock(&i7core_edac_lock);
2438                 return;
2439         }
2440
2441         list_for_each_entry(i7core_dev, &i7core_edac_list, list)
2442                 i7core_unregister_mci(i7core_dev);
2443
2444         /* Release PCI resources */
2445         i7core_put_all_devices();
2446
2447         probed--;
2448
2449         mutex_unlock(&i7core_edac_lock);
2450 }
2451
2452 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2453
2454 /*
2455  *      i7core_driver   pci_driver structure for this module
2456  *
2457  */
2458 static struct pci_driver i7core_driver = {
2459         .name     = "i7core_edac",
2460         .probe    = i7core_probe,
2461         .remove   = __devexit_p(i7core_remove),
2462         .id_table = i7core_pci_tbl,
2463 };
2464
2465 /*
2466  *      i7core_init             Module entry function
2467  *                      Try to initialize this module for its devices
2468  */
2469 static int __init i7core_init(void)
2470 {
2471         int pci_rc;
2472
2473         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2474
2475         /* Ensure that the OPSTATE is set correctly for POLL or NMI */
2476         opstate_init();
2477
2478         if (use_pci_fixup)
2479                 i7core_xeon_pci_fixup(pci_dev_table);
2480
2481         pci_rc = pci_register_driver(&i7core_driver);
2482
2483         if (pci_rc >= 0)
2484                 return 0;
2485
2486         i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2487                       pci_rc);
2488
2489         return pci_rc;
2490 }
2491
2492 /*
2493  *      i7core_exit()   Module exit function
2494  *                      Unregister the driver
2495  */
2496 static void __exit i7core_exit(void)
2497 {
2498         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2499         pci_unregister_driver(&i7core_driver);
2500 }
2501
2502 module_init(i7core_init);
2503 module_exit(i7core_exit);
2504
2505 MODULE_LICENSE("GPL");
2506 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
2507 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
2508 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2509                    I7CORE_REVISION);
2510
2511 module_param(edac_op_state, int, 0444);
2512 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");