drivers/edac/i7core_edac.c

   1 /* Intel i7 core/Nehalem Memory Controller kernel module
   2  *
   3  * This driver supports yhe memory controllers found on the Intel
   4  * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
   5  * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
   6  * and Westmere-EP.
   7  *
   8  * This file may be distributed under the terms of the
   9  * GNU General Public License version 2 only.
  10  *
  11  * Copyright (c) 2009-2010 by:
  12  *       Mauro Carvalho Chehab <mchehab@redhat.com>
  13  *
  14  * Red Hat Inc. http://www.redhat.com
  15  *
  16  * Forked and adapted from the i5400_edac driver
  17  *
  18  * Based on the following public Intel datasheets:
  19  * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
  20  * Datasheet, Volume 2:
  21  *      http://download.intel.com/design/processor/datashts/320835.pdf
  22  * Intel Xeon Processor 5500 Series Datasheet Volume 2
  23  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
  24  * also available at:
  25  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  26  */
  27
  28 #include <linux/module.h>
  29 #include <linux/init.h>
  30 #include <linux/pci.h>
  31 #include <linux/pci_ids.h>
  32 #include <linux/slab.h>
  33 #include <linux/delay.h>
  34 #include <linux/edac.h>
  35 #include <linux/mmzone.h>
  36 #include <linux/edac_mce.h>
  37 #include <linux/smp.h>
  38 #include <asm/processor.h>
  39
  40 #include "edac_core.h"
  41
  42 /* Static vars */
  43 static LIST_HEAD(i7core_edac_list);
  44 static DEFINE_MUTEX(i7core_edac_lock);
  45 static int probed;
  46
  47 static int use_pci_fixup;
  48 module_param(use_pci_fixup, int, 0444);
  49 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
  50 /*
  51  * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
  52  * registers start at bus 255, and are not reported by BIOS.
  53  * We currently find devices with only 2 sockets. In order to support more QPI
  54  * Quick Path Interconnect, just increment this number.
  55  */
  56 #define MAX_SOCKET_BUSES        2
  57
  58
  59 /*
  60  * Alter this version for the module when modifications are made
  61  */
  62 #define I7CORE_REVISION    " Ver: 1.0.0 " __DATE__
  63 #define EDAC_MOD_STR      "i7core_edac"
  64
  65 /*
  66  * Debug macros
  67  */
  68 #define i7core_printk(level, fmt, arg...)                       \
  69         edac_printk(level, "i7core", fmt, ##arg)
  70
  71 #define i7core_mc_printk(mci, level, fmt, arg...)               \
  72         edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
  73
  74 /*
  75  * i7core Memory Controller Registers
  76  */
  77
  78         /* OFFSETS for Device 0 Function 0 */
  79
  80 #define MC_CFG_CONTROL  0x90
  81
  82         /* OFFSETS for Device 3 Function 0 */
  83
  84 #define MC_CONTROL      0x48
  85 #define MC_STATUS       0x4c
  86 #define MC_MAX_DOD      0x64
  87
  88 /*
  89  * OFFSETS for Device 3 Function 4, as inicated on Xeon 5500 datasheet:
  90  * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
  91  */
  92
  93 #define MC_TEST_ERR_RCV1        0x60
  94   #define DIMM2_COR_ERR(r)                      ((r) & 0x7fff)
  95
  96 #define MC_TEST_ERR_RCV0        0x64
  97   #define DIMM1_COR_ERR(r)                      (((r) >> 16) & 0x7fff)
  98   #define DIMM0_COR_ERR(r)                      ((r) & 0x7fff)
  99
 100 /* OFFSETS for Device 3 Function 2, as inicated on Xeon 5500 datasheet */
 101 #define MC_COR_ECC_CNT_0        0x80
 102 #define MC_COR_ECC_CNT_1        0x84
 103 #define MC_COR_ECC_CNT_2        0x88
 104 #define MC_COR_ECC_CNT_3        0x8c
 105 #define MC_COR_ECC_CNT_4        0x90
 106 #define MC_COR_ECC_CNT_5        0x94
 107
 108 #define DIMM_TOP_COR_ERR(r)                     (((r) >> 16) & 0x7fff)
 109 #define DIMM_BOT_COR_ERR(r)                     ((r) & 0x7fff)
 110
 111
 112         /* OFFSETS for Devices 4,5 and 6 Function 0 */
 113
 114 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
 115   #define THREE_DIMMS_PRESENT           (1 << 24)
 116   #define SINGLE_QUAD_RANK_PRESENT      (1 << 23)
 117   #define QUAD_RANK_PRESENT             (1 << 22)
 118   #define REGISTERED_DIMM               (1 << 15)
 119
 120 #define MC_CHANNEL_MAPPER       0x60
 121   #define RDLCH(r, ch)          ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
 122   #define WRLCH(r, ch)          ((((r) >> (ch * 6)) & 0x07) - 1)
 123
 124 #define MC_CHANNEL_RANK_PRESENT 0x7c
 125   #define RANK_PRESENT_MASK             0xffff
 126
 127 #define MC_CHANNEL_ADDR_MATCH   0xf0
 128 #define MC_CHANNEL_ERROR_MASK   0xf8
 129 #define MC_CHANNEL_ERROR_INJECT 0xfc
 130   #define INJECT_ADDR_PARITY    0x10
 131   #define INJECT_ECC            0x08
 132   #define MASK_CACHELINE        0x06
 133   #define MASK_FULL_CACHELINE   0x06
 134   #define MASK_MSB32_CACHELINE  0x04
 135   #define MASK_LSB32_CACHELINE  0x02
 136   #define NO_MASK_CACHELINE     0x00
 137   #define REPEAT_EN             0x01
 138
 139         /* OFFSETS for Devices 4,5 and 6 Function 1 */
 140
 141 #define MC_DOD_CH_DIMM0         0x48
 142 #define MC_DOD_CH_DIMM1         0x4c
 143 #define MC_DOD_CH_DIMM2         0x50
 144   #define RANKOFFSET_MASK       ((1 << 12) | (1 << 11) | (1 << 10))
 145   #define RANKOFFSET(x)         ((x & RANKOFFSET_MASK) >> 10)
 146   #define DIMM_PRESENT_MASK     (1 << 9)
 147   #define DIMM_PRESENT(x)       (((x) & DIMM_PRESENT_MASK) >> 9)
 148   #define MC_DOD_NUMBANK_MASK           ((1 << 8) | (1 << 7))
 149   #define MC_DOD_NUMBANK(x)             (((x) & MC_DOD_NUMBANK_MASK) >> 7)
 150   #define MC_DOD_NUMRANK_MASK           ((1 << 6) | (1 << 5))
 151   #define MC_DOD_NUMRANK(x)             (((x) & MC_DOD_NUMRANK_MASK) >> 5)
 152   #define MC_DOD_NUMROW_MASK            ((1 << 4) | (1 << 3) | (1 << 2))
 153   #define MC_DOD_NUMROW(x)              (((x) & MC_DOD_NUMROW_MASK) >> 2)
 154   #define MC_DOD_NUMCOL_MASK            3
 155   #define MC_DOD_NUMCOL(x)              ((x) & MC_DOD_NUMCOL_MASK)
 156
 157 #define MC_RANK_PRESENT         0x7c
 158
 159 #define MC_SAG_CH_0     0x80
 160 #define MC_SAG_CH_1     0x84
 161 #define MC_SAG_CH_2     0x88
 162 #define MC_SAG_CH_3     0x8c
 163 #define MC_SAG_CH_4     0x90
 164 #define MC_SAG_CH_5     0x94
 165 #define MC_SAG_CH_6     0x98
 166 #define MC_SAG_CH_7     0x9c
 167
 168 #define MC_RIR_LIMIT_CH_0       0x40
 169 #define MC_RIR_LIMIT_CH_1       0x44
 170 #define MC_RIR_LIMIT_CH_2       0x48
 171 #define MC_RIR_LIMIT_CH_3       0x4C
 172 #define MC_RIR_LIMIT_CH_4       0x50
 173 #define MC_RIR_LIMIT_CH_5       0x54
 174 #define MC_RIR_LIMIT_CH_6       0x58
 175 #define MC_RIR_LIMIT_CH_7       0x5C
 176 #define MC_RIR_LIMIT_MASK       ((1 << 10) - 1)
 177
 178 #define MC_RIR_WAY_CH           0x80
 179   #define MC_RIR_WAY_OFFSET_MASK        (((1 << 14) - 1) & ~0x7)
 180   #define MC_RIR_WAY_RANK_MASK          0x7
 181
 182 /*
 183  * i7core structs
 184  */
 185
 186 #define NUM_CHANS 3
 187 #define MAX_DIMMS 3             /* Max DIMMS per channel */
 188 #define MAX_MCR_FUNC  4
 189 #define MAX_CHAN_FUNC 3
 190
 191 struct i7core_info {
 192         u32     mc_control;
 193         u32     mc_status;
 194         u32     max_dod;
 195         u32     ch_map;
 196 };
 197
 198
 199 struct i7core_inject {
 200         int     enable;
 201
 202         u32     section;
 203         u32     type;
 204         u32     eccmask;
 205
 206         /* Error address mask */
 207         int channel, dimm, rank, bank, page, col;
 208 };
 209
 210 struct i7core_channel {
 211         u32             ranks;
 212         u32             dimms;
 213 };
 214
 215 struct pci_id_descr {
 216         int                     dev;
 217         int                     func;
 218         int                     dev_id;
 219         int                     optional;
 220 };
 221
 222 struct pci_id_table {
 223         const struct pci_id_descr       *descr;
 224         int                             n_devs;
 225 };
 226
 227 struct i7core_dev {
 228         struct list_head        list;
 229         u8                      socket;
 230         struct pci_dev          **pdev;
 231         int                     n_devs;
 232         struct mem_ctl_info     *mci;
 233 };
 234
 235 struct i7core_pvt {
 236         struct pci_dev  *pci_noncore;
 237         struct pci_dev  *pci_mcr[MAX_MCR_FUNC + 1];
 238         struct pci_dev  *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
 239
 240         struct i7core_dev *i7core_dev;
 241
 242         struct i7core_info      info;
 243         struct i7core_inject    inject;
 244         struct i7core_channel   channel[NUM_CHANS];
 245
 246         int             channels; /* Number of active channels */
 247
 248         int             ce_count_available;
 249         int             csrow_map[NUM_CHANS][MAX_DIMMS];
 250
 251                         /* ECC corrected errors counts per udimm */
 252         unsigned long   udimm_ce_count[MAX_DIMMS];
 253         int             udimm_last_ce_count[MAX_DIMMS];
 254                         /* ECC corrected errors counts per rdimm */
 255         unsigned long   rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
 256         int             rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
 257
 258         unsigned int    is_registered;
 259
 260         /* mcelog glue */
 261         struct edac_mce         edac_mce;
 262
 263         /* Fifo double buffers */
 264         struct mce              mce_entry[MCE_LOG_LEN];
 265         struct mce              mce_outentry[MCE_LOG_LEN];
 266
 267         /* Fifo in/out counters */
 268         unsigned                mce_in, mce_out;
 269
 270         /* Count indicator to show errors not got */
 271         unsigned                mce_overrun;
 272
 273         /* Struct to control EDAC polling */
 274         struct edac_pci_ctl_info *i7core_pci;
 275 };
 276
 277 #define PCI_DESCR(device, function, device_id)  \
 278         .dev = (device),                        \
 279         .func = (function),                     \
 280         .dev_id = (device_id)
 281
 282 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
 283                 /* Memory controller */
 284         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
 285         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
 286                         /* Exists only for RDIMM */
 287         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1  },
 288         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
 289
 290                 /* Channel 0 */
 291         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
 292         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
 293         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
 294         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
 295
 296                 /* Channel 1 */
 297         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
 298         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
 299         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
 300         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
 301
 302                 /* Channel 2 */
 303         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
 304         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
 305         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
 306         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
 307
 308                 /* Generic Non-core registers */
 309         /*
 310          * This is the PCI device on i7core and on Xeon 35xx (8086:2c41)
 311          * On Xeon 55xx, however, it has a different id (8086:2c40). So,
 312          * the probing code needs to test for the other address in case of
 313          * failure of this one
 314          */
 315         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE)  },
 316
 317 };
 318
 319 static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
 320         { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR)         },
 321         { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD)      },
 322         { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST)     },
 323
 324         { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
 325         { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
 326         { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
 327         { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC)   },
 328
 329         { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
 330         { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
 331         { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
 332         { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC)   },
 333
 334         /*
 335          * This is the PCI device has an alternate address on some
 336          * processors like Core i7 860
 337          */
 338         { PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE)     },
 339 };
 340
 341 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
 342                 /* Memory controller */
 343         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2)     },
 344         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2)  },
 345                         /* Exists only for RDIMM */
 346         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1  },
 347         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
 348
 349                 /* Channel 0 */
 350         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
 351         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
 352         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
 353         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2)   },
 354
 355                 /* Channel 1 */
 356         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
 357         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
 358         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
 359         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2)   },
 360
 361                 /* Channel 2 */
 362         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
 363         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
 364         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
 365         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2)   },
 366
 367                 /* Generic Non-core registers */
 368         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2)  },
 369
 370 };
 371
 372 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
 373 static const struct pci_id_table pci_dev_table[] = {
 374         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
 375         PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
 376         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
 377 };
 378
 379 /*
 380  *      pci_device_id   table for which devices we are looking for
 381  */
 382 static const struct pci_device_id i7core_pci_tbl[] __devinitdata = {
 383         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
 384         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
 385         {0,}                    /* 0 terminated list. */
 386 };
 387
 388 /****************************************************************************
 389                         Anciliary status routines
 390  ****************************************************************************/
 391
 392         /* MC_CONTROL bits */
 393 #define CH_ACTIVE(pvt, ch)      ((pvt)->info.mc_control & (1 << (8 + ch)))
 394 #define ECCx8(pvt)              ((pvt)->info.mc_control & (1 << 1))
 395
 396         /* MC_STATUS bits */
 397 #define ECC_ENABLED(pvt)        ((pvt)->info.mc_status & (1 << 4))
 398 #define CH_DISABLED(pvt, ch)    ((pvt)->info.mc_status & (1 << ch))
 399
 400         /* MC_MAX_DOD read functions */
 401 static inline int numdimms(u32 dimms)
 402 {
 403         return (dimms & 0x3) + 1;
 404 }
 405
 406 static inline int numrank(u32 rank)
 407 {
 408         static int ranks[4] = { 1, 2, 4, -EINVAL };
 409
 410         return ranks[rank & 0x3];
 411 }
 412
 413 static inline int numbank(u32 bank)
 414 {
 415         static int banks[4] = { 4, 8, 16, -EINVAL };
 416
 417         return banks[bank & 0x3];
 418 }
 419
 420 static inline int numrow(u32 row)
 421 {
 422         static int rows[8] = {
 423                 1 << 12, 1 << 13, 1 << 14, 1 << 15,
 424                 1 << 16, -EINVAL, -EINVAL, -EINVAL,
 425         };
 426
 427         return rows[row & 0x7];
 428 }
 429
 430 static inline int numcol(u32 col)
 431 {
 432         static int cols[8] = {
 433                 1 << 10, 1 << 11, 1 << 12, -EINVAL,
 434         };
 435         return cols[col & 0x3];
 436 }
 437
 438 static struct i7core_dev *get_i7core_dev(u8 socket)
 439 {
 440         struct i7core_dev *i7core_dev;
 441
 442         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
 443                 if (i7core_dev->socket == socket)
 444                         return i7core_dev;
 445         }
 446
 447         return NULL;
 448 }
 449
 450 /****************************************************************************
 451                         Memory check routines
 452  ****************************************************************************/
 453 static struct pci_dev *get_pdev_slot_func(u8 socket, unsigned slot,
 454                                           unsigned func)
 455 {
 456         struct i7core_dev *i7core_dev = get_i7core_dev(socket);
 457         int i;
 458
 459         if (!i7core_dev)
 460                 return NULL;
 461
 462         for (i = 0; i < i7core_dev->n_devs; i++) {
 463                 if (!i7core_dev->pdev[i])
 464                         continue;
 465
 466                 if (PCI_SLOT(i7core_dev->pdev[i]->devfn) == slot &&
 467                     PCI_FUNC(i7core_dev->pdev[i]->devfn) == func) {
 468                         return i7core_dev->pdev[i];
 469                 }
 470         }
 471
 472         return NULL;
 473 }
 474
 475 /**
 476  * i7core_get_active_channels() - gets the number of channels and csrows
 477  * @socket:     Quick Path Interconnect socket
 478  * @channels:   Number of channels that will be returned
 479  * @csrows:     Number of csrows found
 480  *
 481  * Since EDAC core needs to know in advance the number of available channels
 482  * and csrows, in order to allocate memory for csrows/channels, it is needed
 483  * to run two similar steps. At the first step, implemented on this function,
 484  * it checks the number of csrows/channels present at one socket.
 485  * this is used in order to properly allocate the size of mci components.
 486  *
 487  * It should be noticed that none of the current available datasheets explain
 488  * or even mention how csrows are seen by the memory controller. So, we need
 489  * to add a fake description for csrows.
 490  * So, this driver is attributing one DIMM memory for one csrow.
 491  */
 492 static int i7core_get_active_channels(const u8 socket, unsigned *channels,
 493                                       unsigned *csrows)
 494 {
 495         struct pci_dev *pdev = NULL;
 496         int i, j;
 497         u32 status, control;
 498
 499         *channels = 0;
 500         *csrows = 0;
 501
 502         pdev = get_pdev_slot_func(socket, 3, 0);
 503         if (!pdev) {
 504                 i7core_printk(KERN_ERR, "Couldn't find socket %d fn 3.0!!!\n",
 505                               socket);
 506                 return -ENODEV;
 507         }
 508
 509         /* Device 3 function 0 reads */
 510         pci_read_config_dword(pdev, MC_STATUS, &status);
 511         pci_read_config_dword(pdev, MC_CONTROL, &control);
 512
 513         for (i = 0; i < NUM_CHANS; i++) {
 514                 u32 dimm_dod[3];
 515                 /* Check if the channel is active */
 516                 if (!(control & (1 << (8 + i))))
 517                         continue;
 518
 519                 /* Check if the channel is disabled */
 520                 if (status & (1 << i))
 521                         continue;
 522
 523                 pdev = get_pdev_slot_func(socket, i + 4, 1);
 524                 if (!pdev) {
 525                         i7core_printk(KERN_ERR, "Couldn't find socket %d "
 526                                                 "fn %d.%d!!!\n",
 527                                                 socket, i + 4, 1);
 528                         return -ENODEV;
 529                 }
 530                 /* Devices 4-6 function 1 */
 531                 pci_read_config_dword(pdev,
 532                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 533                 pci_read_config_dword(pdev,
 534                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 535                 pci_read_config_dword(pdev,
 536                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 537
 538                 (*channels)++;
 539
 540                 for (j = 0; j < 3; j++) {
 541                         if (!DIMM_PRESENT(dimm_dod[j]))
 542                                 continue;
 543                         (*csrows)++;
 544                 }
 545         }
 546
 547         debugf0("Number of active channels on socket %d: %d\n",
 548                 socket, *channels);
 549
 550         return 0;
 551 }
 552
 553 static int get_dimm_config(const struct mem_ctl_info *mci, int *csrow)
 554 {
 555         struct i7core_pvt *pvt = mci->pvt_info;
 556         struct csrow_info *csr;
 557         struct pci_dev *pdev;
 558         int i, j;
 559         unsigned long last_page = 0;
 560         enum edac_type mode;
 561         enum mem_type mtype;
 562
 563         /* Get data from the MC register, function 0 */
 564         pdev = pvt->pci_mcr[0];
 565         if (!pdev)
 566                 return -ENODEV;
 567
 568         /* Device 3 function 0 reads */
 569         pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
 570         pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
 571         pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
 572         pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
 573
 574         debugf0("QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
 575                 pvt->i7core_dev->socket, pvt->info.mc_control, pvt->info.mc_status,
 576                 pvt->info.max_dod, pvt->info.ch_map);
 577
 578         if (ECC_ENABLED(pvt)) {
 579                 debugf0("ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
 580                 if (ECCx8(pvt))
 581                         mode = EDAC_S8ECD8ED;
 582                 else
 583                         mode = EDAC_S4ECD4ED;
 584         } else {
 585                 debugf0("ECC disabled\n");
 586                 mode = EDAC_NONE;
 587         }
 588
 589         /* FIXME: need to handle the error codes */
 590         debugf0("DOD Max limits: DIMMS: %d, %d-ranked, %d-banked "
 591                 "x%x x 0x%x\n",
 592                 numdimms(pvt->info.max_dod),
 593                 numrank(pvt->info.max_dod >> 2),
 594                 numbank(pvt->info.max_dod >> 4),
 595                 numrow(pvt->info.max_dod >> 6),
 596                 numcol(pvt->info.max_dod >> 9));
 597
 598         for (i = 0; i < NUM_CHANS; i++) {
 599                 u32 data, dimm_dod[3], value[8];
 600
 601                 if (!pvt->pci_ch[i][0])
 602                         continue;
 603
 604                 if (!CH_ACTIVE(pvt, i)) {
 605                         debugf0("Channel %i is not active\n", i);
 606                         continue;
 607                 }
 608                 if (CH_DISABLED(pvt, i)) {
 609                         debugf0("Channel %i is disabled\n", i);
 610                         continue;
 611                 }
 612
 613                 /* Devices 4-6 function 0 */
 614                 pci_read_config_dword(pvt->pci_ch[i][0],
 615                                 MC_CHANNEL_DIMM_INIT_PARAMS, &data);
 616
 617                 pvt->channel[i].ranks = (data & QUAD_RANK_PRESENT) ?
 618                                                 4 : 2;
 619
 620                 if (data & REGISTERED_DIMM)
 621                         mtype = MEM_RDDR3;
 622                 else
 623                         mtype = MEM_DDR3;
 624 #if 0
 625                 if (data & THREE_DIMMS_PRESENT)
 626                         pvt->channel[i].dimms = 3;
 627                 else if (data & SINGLE_QUAD_RANK_PRESENT)
 628                         pvt->channel[i].dimms = 1;
 629                 else
 630                         pvt->channel[i].dimms = 2;
 631 #endif
 632
 633                 /* Devices 4-6 function 1 */
 634                 pci_read_config_dword(pvt->pci_ch[i][1],
 635                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
 636                 pci_read_config_dword(pvt->pci_ch[i][1],
 637                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
 638                 pci_read_config_dword(pvt->pci_ch[i][1],
 639                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
 640
 641                 debugf0("Ch%d phy rd%d, wr%d (0x%08x): "
 642                         "%d ranks, %cDIMMs\n",
 643                         i,
 644                         RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
 645                         data,
 646                         pvt->channel[i].ranks,
 647                         (data & REGISTERED_DIMM) ? 'R' : 'U');
 648
 649                 for (j = 0; j < 3; j++) {
 650                         u32 banks, ranks, rows, cols;
 651                         u32 size, npages;
 652
 653                         if (!DIMM_PRESENT(dimm_dod[j]))
 654                                 continue;
 655
 656                         banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
 657                         ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
 658                         rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
 659                         cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
 660
 661                         /* DDR3 has 8 I/O banks */
 662                         size = (rows * cols * banks * ranks) >> (20 - 3);
 663
 664                         pvt->channel[i].dimms++;
 665
 666                         debugf0("\tdimm %d %d Mb offset: %x, "
 667                                 "bank: %d, rank: %d, row: %#x, col: %#x\n",
 668                                 j, size,
 669                                 RANKOFFSET(dimm_dod[j]),
 670                                 banks, ranks, rows, cols);
 671
 672                         npages = MiB_TO_PAGES(size);
 673
 674                         csr = &mci->csrows[*csrow];
 675                         csr->first_page = last_page + 1;
 676                         last_page += npages;
 677                         csr->last_page = last_page;
 678                         csr->nr_pages = npages;
 679
 680                         csr->page_mask = 0;
 681                         csr->grain = 8;
 682                         csr->csrow_idx = *csrow;
 683                         csr->nr_channels = 1;
 684
 685                         csr->channels[0].chan_idx = i;
 686                         csr->channels[0].ce_count = 0;
 687
 688                         pvt->csrow_map[i][j] = *csrow;
 689
 690                         switch (banks) {
 691                         case 4:
 692                                 csr->dtype = DEV_X4;
 693                                 break;
 694                         case 8:
 695                                 csr->dtype = DEV_X8;
 696                                 break;
 697                         case 16:
 698                                 csr->dtype = DEV_X16;
 699                                 break;
 700                         default:
 701                                 csr->dtype = DEV_UNKNOWN;
 702                         }
 703
 704                         csr->edac_mode = mode;
 705                         csr->mtype = mtype;
 706
 707                         (*csrow)++;
 708                 }
 709
 710                 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
 711                 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
 712                 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
 713                 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
 714                 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
 715                 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
 716                 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
 717                 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
 718                 debugf1("\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
 719                 for (j = 0; j < 8; j++)
 720                         debugf1("\t\t%#x\t%#x\t%#x\n",
 721                                 (value[j] >> 27) & 0x1,
 722                                 (value[j] >> 24) & 0x7,
 723                                 (value[j] && ((1 << 24) - 1)));
 724         }
 725
 726         return 0;
 727 }
 728
 729 /****************************************************************************
 730                         Error insertion routines
 731  ****************************************************************************/
 732
 733 /* The i7core has independent error injection features per channel.
 734    However, to have a simpler code, we don't allow enabling error injection
 735    on more than one channel.
 736    Also, since a change at an inject parameter will be applied only at enable,
 737    we're disabling error injection on all write calls to the sysfs nodes that
 738    controls the error code injection.
 739  */
 740 static int disable_inject(const struct mem_ctl_info *mci)
 741 {
 742         struct i7core_pvt *pvt = mci->pvt_info;
 743
 744         pvt->inject.enable = 0;
 745
 746         if (!pvt->pci_ch[pvt->inject.channel][0])
 747                 return -ENODEV;
 748
 749         pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
 750                                 MC_CHANNEL_ERROR_INJECT, 0);
 751
 752         return 0;
 753 }
 754
 755 /*
 756  * i7core inject inject.section
 757  *
 758  *      accept and store error injection inject.section value
 759  *      bit 0 - refers to the lower 32-byte half cacheline
 760  *      bit 1 - refers to the upper 32-byte half cacheline
 761  */
 762 static ssize_t i7core_inject_section_store(struct mem_ctl_info *mci,
 763                                            const char *data, size_t count)
 764 {
 765         struct i7core_pvt *pvt = mci->pvt_info;
 766         unsigned long value;
 767         int rc;
 768
 769         if (pvt->inject.enable)
 770                 disable_inject(mci);
 771
 772         rc = strict_strtoul(data, 10, &value);
 773         if ((rc < 0) || (value > 3))
 774                 return -EIO;
 775
 776         pvt->inject.section = (u32) value;
 777         return count;
 778 }
 779
 780 static ssize_t i7core_inject_section_show(struct mem_ctl_info *mci,
 781                                               char *data)
 782 {
 783         struct i7core_pvt *pvt = mci->pvt_info;
 784         return sprintf(data, "0x%08x\n", pvt->inject.section);
 785 }
 786
 787 /*
 788  * i7core inject.type
 789  *
 790  *      accept and store error injection inject.section value
 791  *      bit 0 - repeat enable - Enable error repetition
 792  *      bit 1 - inject ECC error
 793  *      bit 2 - inject parity error
 794  */
 795 static ssize_t i7core_inject_type_store(struct mem_ctl_info *mci,
 796                                         const char *data, size_t count)
 797 {
 798         struct i7core_pvt *pvt = mci->pvt_info;
 799         unsigned long value;
 800         int rc;
 801
 802         if (pvt->inject.enable)
 803                 disable_inject(mci);
 804
 805         rc = strict_strtoul(data, 10, &value);
 806         if ((rc < 0) || (value > 7))
 807                 return -EIO;
 808
 809         pvt->inject.type = (u32) value;
 810         return count;
 811 }
 812
 813 static ssize_t i7core_inject_type_show(struct mem_ctl_info *mci,
 814                                               char *data)
 815 {
 816         struct i7core_pvt *pvt = mci->pvt_info;
 817         return sprintf(data, "0x%08x\n", pvt->inject.type);
 818 }
 819
 820 /*
 821  * i7core_inject_inject.eccmask_store
 822  *
 823  * The type of error (UE/CE) will depend on the inject.eccmask value:
 824  *   Any bits set to a 1 will flip the corresponding ECC bit
 825  *   Correctable errors can be injected by flipping 1 bit or the bits within
 826  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 827  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 828  *   uncorrectable error to be injected.
 829  */
 830 static ssize_t i7core_inject_eccmask_store(struct mem_ctl_info *mci,
 831                                         const char *data, size_t count)
 832 {
 833         struct i7core_pvt *pvt = mci->pvt_info;
 834         unsigned long value;
 835         int rc;
 836
 837         if (pvt->inject.enable)
 838                 disable_inject(mci);
 839
 840         rc = strict_strtoul(data, 10, &value);
 841         if (rc < 0)
 842                 return -EIO;
 843
 844         pvt->inject.eccmask = (u32) value;
 845         return count;
 846 }
 847
 848 static ssize_t i7core_inject_eccmask_show(struct mem_ctl_info *mci,
 849                                               char *data)
 850 {
 851         struct i7core_pvt *pvt = mci->pvt_info;
 852         return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
 853 }
 854
 855 /*
 856  * i7core_addrmatch
 857  *
 858  * The type of error (UE/CE) will depend on the inject.eccmask value:
 859  *   Any bits set to a 1 will flip the corresponding ECC bit
 860  *   Correctable errors can be injected by flipping 1 bit or the bits within
 861  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
 862  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
 863  *   uncorrectable error to be injected.
 864  */
 865
 866 #define DECLARE_ADDR_MATCH(param, limit)                        \
 867 static ssize_t i7core_inject_store_##param(                     \
 868                 struct mem_ctl_info *mci,                       \
 869                 const char *data, size_t count)                 \
 870 {                                                               \
 871         struct i7core_pvt *pvt;                                 \
 872         long value;                                             \
 873         int rc;                                                 \
 874                                                                 \
 875         debugf1("%s()\n", __func__);                            \
 876         pvt = mci->pvt_info;                                    \
 877                                                                 \
 878         if (pvt->inject.enable)                                 \
 879                 disable_inject(mci);                            \
 880                                                                 \
 881         if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
 882                 value = -1;                                     \
 883         else {                                                  \
 884                 rc = strict_strtoul(data, 10, &value);          \
 885                 if ((rc < 0) || (value >= limit))               \
 886                         return -EIO;                            \
 887         }                                                       \
 888                                                                 \
 889         pvt->inject.param = value;                              \
 890                                                                 \
 891         return count;                                           \
 892 }                                                               \
 893                                                                 \
 894 static ssize_t i7core_inject_show_##param(                      \
 895                 struct mem_ctl_info *mci,                       \
 896                 char *data)                                     \
 897 {                                                               \
 898         struct i7core_pvt *pvt;                                 \
 899                                                                 \
 900         pvt = mci->pvt_info;                                    \
 901         debugf1("%s() pvt=%p\n", __func__, pvt);                \
 902         if (pvt->inject.param < 0)                              \
 903                 return sprintf(data, "any\n");                  \
 904         else                                                    \
 905                 return sprintf(data, "%d\n", pvt->inject.param);\
 906 }
 907
 908 #define ATTR_ADDR_MATCH(param)                                  \
 909         {                                                       \
 910                 .attr = {                                       \
 911                         .name = #param,                         \
 912                         .mode = (S_IRUGO | S_IWUSR)             \
 913                 },                                              \
 914                 .show  = i7core_inject_show_##param,            \
 915                 .store = i7core_inject_store_##param,           \
 916         }
 917
 918 DECLARE_ADDR_MATCH(channel, 3);
 919 DECLARE_ADDR_MATCH(dimm, 3);
 920 DECLARE_ADDR_MATCH(rank, 4);
 921 DECLARE_ADDR_MATCH(bank, 32);
 922 DECLARE_ADDR_MATCH(page, 0x10000);
 923 DECLARE_ADDR_MATCH(col, 0x4000);
 924
 925 static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
 926 {
 927         u32 read;
 928         int count;
 929
 930         debugf0("setting pci %02x:%02x.%x reg=%02x value=%08x\n",
 931                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 932                 where, val);
 933
 934         for (count = 0; count < 10; count++) {
 935                 if (count)
 936                         msleep(100);
 937                 pci_write_config_dword(dev, where, val);
 938                 pci_read_config_dword(dev, where, &read);
 939
 940                 if (read == val)
 941                         return 0;
 942         }
 943
 944         i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
 945                 "write=%08x. Read=%08x\n",
 946                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
 947                 where, val, read);
 948
 949         return -EINVAL;
 950 }
 951
 952 /*
 953  * This routine prepares the Memory Controller for error injection.
 954  * The error will be injected when some process tries to write to the
 955  * memory that matches the given criteria.
 956  * The criteria can be set in terms of a mask where dimm, rank, bank, page
 957  * and col can be specified.
 958  * A -1 value for any of the mask items will make the MCU to ignore
 959  * that matching criteria for error injection.
 960  *
 961  * It should be noticed that the error will only happen after a write operation
 962  * on a memory that matches the condition. if REPEAT_EN is not enabled at
 963  * inject mask, then it will produce just one error. Otherwise, it will repeat
 964  * until the injectmask would be cleaned.
 965  *
 966  * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
 967  *    is reliable enough to check if the MC is using the
 968  *    three channels. However, this is not clear at the datasheet.
 969  */
 970 static ssize_t i7core_inject_enable_store(struct mem_ctl_info *mci,
 971                                        const char *data, size_t count)
 972 {
 973         struct i7core_pvt *pvt = mci->pvt_info;
 974         u32 injectmask;
 975         u64 mask = 0;
 976         int  rc;
 977         long enable;
 978
 979         if (!pvt->pci_ch[pvt->inject.channel][0])
 980                 return 0;
 981
 982         rc = strict_strtoul(data, 10, &enable);
 983         if ((rc < 0))
 984                 return 0;
 985
 986         if (enable) {
 987                 pvt->inject.enable = 1;
 988         } else {
 989                 disable_inject(mci);
 990                 return count;
 991         }
 992
 993         /* Sets pvt->inject.dimm mask */
 994         if (pvt->inject.dimm < 0)
 995                 mask |= 1LL << 41;
 996         else {
 997                 if (pvt->channel[pvt->inject.channel].dimms > 2)
 998                         mask |= (pvt->inject.dimm & 0x3LL) << 35;
 999                 else
1000                         mask |= (pvt->inject.dimm & 0x1LL) << 36;
1001         }
1002
1003         /* Sets pvt->inject.rank mask */
1004         if (pvt->inject.rank < 0)
1005                 mask |= 1LL << 40;
1006         else {
1007                 if (pvt->channel[pvt->inject.channel].dimms > 2)
1008                         mask |= (pvt->inject.rank & 0x1LL) << 34;
1009                 else
1010                         mask |= (pvt->inject.rank & 0x3LL) << 34;
1011         }
1012
1013         /* Sets pvt->inject.bank mask */
1014         if (pvt->inject.bank < 0)
1015                 mask |= 1LL << 39;
1016         else
1017                 mask |= (pvt->inject.bank & 0x15LL) << 30;
1018
1019         /* Sets pvt->inject.page mask */
1020         if (pvt->inject.page < 0)
1021                 mask |= 1LL << 38;
1022         else
1023                 mask |= (pvt->inject.page & 0xffff) << 14;
1024
1025         /* Sets pvt->inject.column mask */
1026         if (pvt->inject.col < 0)
1027                 mask |= 1LL << 37;
1028         else
1029                 mask |= (pvt->inject.col & 0x3fff);
1030
1031         /*
1032          * bit    0: REPEAT_EN
1033          * bits 1-2: MASK_HALF_CACHELINE
1034          * bit    3: INJECT_ECC
1035          * bit    4: INJECT_ADDR_PARITY
1036          */
1037
1038         injectmask = (pvt->inject.type & 1) |
1039                      (pvt->inject.section & 0x3) << 1 |
1040                      (pvt->inject.type & 0x6) << (3 - 1);
1041
1042         /* Unlock writes to registers - this register is write only */
1043         pci_write_config_dword(pvt->pci_noncore,
1044                                MC_CFG_CONTROL, 0x2);
1045
1046         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1047                                MC_CHANNEL_ADDR_MATCH, mask);
1048         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1049                                MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
1050
1051         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1052                                MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
1053
1054         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1055                                MC_CHANNEL_ERROR_INJECT, injectmask);
1056
1057         /*
1058          * This is something undocumented, based on my tests
1059          * Without writing 8 to this register, errors aren't injected. Not sure
1060          * why.
1061          */
1062         pci_write_config_dword(pvt->pci_noncore,
1063                                MC_CFG_CONTROL, 8);
1064
1065         debugf0("Error inject addr match 0x%016llx, ecc 0x%08x,"
1066                 " inject 0x%08x\n",
1067                 mask, pvt->inject.eccmask, injectmask);
1068
1069
1070         return count;
1071 }
1072
1073 static ssize_t i7core_inject_enable_show(struct mem_ctl_info *mci,
1074                                         char *data)
1075 {
1076         struct i7core_pvt *pvt = mci->pvt_info;
1077         u32 injectmask;
1078
1079         if (!pvt->pci_ch[pvt->inject.channel][0])
1080                 return 0;
1081
1082         pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1083                                MC_CHANNEL_ERROR_INJECT, &injectmask);
1084
1085         debugf0("Inject error read: 0x%018x\n", injectmask);
1086
1087         if (injectmask & 0x0c)
1088                 pvt->inject.enable = 1;
1089
1090         return sprintf(data, "%d\n", pvt->inject.enable);
1091 }
1092
1093 #define DECLARE_COUNTER(param)                                  \
1094 static ssize_t i7core_show_counter_##param(                     \
1095                 struct mem_ctl_info *mci,                       \
1096                 char *data)                                     \
1097 {                                                               \
1098         struct i7core_pvt *pvt = mci->pvt_info;                 \
1099                                                                 \
1100         debugf1("%s() \n", __func__);                           \
1101         if (!pvt->ce_count_available || (pvt->is_registered))   \
1102                 return sprintf(data, "data unavailable\n");     \
1103         return sprintf(data, "%lu\n",                           \
1104                         pvt->udimm_ce_count[param]);            \
1105 }
1106
1107 #define ATTR_COUNTER(param)                                     \
1108         {                                                       \
1109                 .attr = {                                       \
1110                         .name = __stringify(udimm##param),      \
1111                         .mode = (S_IRUGO | S_IWUSR)             \
1112                 },                                              \
1113                 .show  = i7core_show_counter_##param            \
1114         }
1115
1116 DECLARE_COUNTER(0);
1117 DECLARE_COUNTER(1);
1118 DECLARE_COUNTER(2);
1119
1120 /*
1121  * Sysfs struct
1122  */
1123
1124 static const struct mcidev_sysfs_attribute i7core_addrmatch_attrs[] = {
1125         ATTR_ADDR_MATCH(channel),
1126         ATTR_ADDR_MATCH(dimm),
1127         ATTR_ADDR_MATCH(rank),
1128         ATTR_ADDR_MATCH(bank),
1129         ATTR_ADDR_MATCH(page),
1130         ATTR_ADDR_MATCH(col),
1131         { } /* End of list */
1132 };
1133
1134 static const struct mcidev_sysfs_group i7core_inject_addrmatch = {
1135         .name  = "inject_addrmatch",
1136         .mcidev_attr = i7core_addrmatch_attrs,
1137 };
1138
1139 static const struct mcidev_sysfs_attribute i7core_udimm_counters_attrs[] = {
1140         ATTR_COUNTER(0),
1141         ATTR_COUNTER(1),
1142         ATTR_COUNTER(2),
1143         { .attr = { .name = NULL } }
1144 };
1145
1146 static const struct mcidev_sysfs_group i7core_udimm_counters = {
1147         .name  = "all_channel_counts",
1148         .mcidev_attr = i7core_udimm_counters_attrs,
1149 };
1150
1151 static const struct mcidev_sysfs_attribute i7core_sysfs_rdimm_attrs[] = {
1152         {
1153                 .attr = {
1154                         .name = "inject_section",
1155                         .mode = (S_IRUGO | S_IWUSR)
1156                 },
1157                 .show  = i7core_inject_section_show,
1158                 .store = i7core_inject_section_store,
1159         }, {
1160                 .attr = {
1161                         .name = "inject_type",
1162                         .mode = (S_IRUGO | S_IWUSR)
1163                 },
1164                 .show  = i7core_inject_type_show,
1165                 .store = i7core_inject_type_store,
1166         }, {
1167                 .attr = {
1168                         .name = "inject_eccmask",
1169                         .mode = (S_IRUGO | S_IWUSR)
1170                 },
1171                 .show  = i7core_inject_eccmask_show,
1172                 .store = i7core_inject_eccmask_store,
1173         }, {
1174                 .grp = &i7core_inject_addrmatch,
1175         }, {
1176                 .attr = {
1177                         .name = "inject_enable",
1178                         .mode = (S_IRUGO | S_IWUSR)
1179                 },
1180                 .show  = i7core_inject_enable_show,
1181                 .store = i7core_inject_enable_store,
1182         },
1183         { }     /* End of list */
1184 };
1185
1186 static const struct mcidev_sysfs_attribute i7core_sysfs_udimm_attrs[] = {
1187         {
1188                 .attr = {
1189                         .name = "inject_section",
1190                         .mode = (S_IRUGO | S_IWUSR)
1191                 },
1192                 .show  = i7core_inject_section_show,
1193                 .store = i7core_inject_section_store,
1194         }, {
1195                 .attr = {
1196                         .name = "inject_type",
1197                         .mode = (S_IRUGO | S_IWUSR)
1198                 },
1199                 .show  = i7core_inject_type_show,
1200                 .store = i7core_inject_type_store,
1201         }, {
1202                 .attr = {
1203                         .name = "inject_eccmask",
1204                         .mode = (S_IRUGO | S_IWUSR)
1205                 },
1206                 .show  = i7core_inject_eccmask_show,
1207                 .store = i7core_inject_eccmask_store,
1208         }, {
1209                 .grp = &i7core_inject_addrmatch,
1210         }, {
1211                 .attr = {
1212                         .name = "inject_enable",
1213                         .mode = (S_IRUGO | S_IWUSR)
1214                 },
1215                 .show  = i7core_inject_enable_show,
1216                 .store = i7core_inject_enable_store,
1217         }, {
1218                 .grp = &i7core_udimm_counters,
1219         },
1220         { }     /* End of list */
1221 };
1222
1223 /****************************************************************************
1224         Device initialization routines: put/get, init/exit
1225  ****************************************************************************/
1226
1227 /*
1228  *      i7core_put_devices      'put' all the devices that we have
1229  *                              reserved via 'get'
1230  */
1231 static void i7core_put_devices(struct i7core_dev *i7core_dev)
1232 {
1233         int i;
1234
1235         debugf0(__FILE__ ": %s()\n", __func__);
1236         for (i = 0; i < i7core_dev->n_devs; i++) {
1237                 struct pci_dev *pdev = i7core_dev->pdev[i];
1238                 if (!pdev)
1239                         continue;
1240                 debugf0("Removing dev %02x:%02x.%d\n",
1241                         pdev->bus->number,
1242                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1243                 pci_dev_put(pdev);
1244         }
1245         kfree(i7core_dev->pdev);
1246 }
1247
1248 static void i7core_put_all_devices(void)
1249 {
1250         struct i7core_dev *i7core_dev, *tmp;
1251
1252         list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1253                 i7core_put_devices(i7core_dev);
1254                 list_del(&i7core_dev->list);
1255                 kfree(i7core_dev);
1256         }
1257 }
1258
1259 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1260 {
1261         struct pci_dev *pdev = NULL;
1262         int i;
1263
1264         /*
1265          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core pci buses
1266          * aren't announced by acpi. So, we need to use a legacy scan probing
1267          * to detect them
1268          */
1269         while (table && table->descr) {
1270                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1271                 if (unlikely(!pdev)) {
1272                         for (i = 0; i < MAX_SOCKET_BUSES; i++)
1273                                 pcibios_scan_specific_bus(255-i);
1274                 }
1275                 pci_dev_put(pdev);
1276                 table++;
1277         }
1278 }
1279
1280 static unsigned i7core_pci_lastbus(void)
1281 {
1282         int last_bus = 0, bus;
1283         struct pci_bus *b = NULL;
1284
1285         while ((b = pci_find_next_bus(b)) != NULL) {
1286                 bus = b->number;
1287                 debugf0("Found bus %d\n", bus);
1288                 if (bus > last_bus)
1289                         last_bus = bus;
1290         }
1291
1292         debugf0("Last bus %d\n", last_bus);
1293
1294         return last_bus;
1295 }
1296
1297 /*
1298  *      i7core_get_devices      Find and perform 'get' operation on the MCH's
1299  *                      device/functions we want to reference for this driver
1300  *
1301  *                      Need to 'get' device 16 func 1 and func 2
1302  */
1303 static int i7core_get_onedevice(struct pci_dev **prev,
1304                                 const struct pci_id_table *table,
1305                                 const unsigned devno,
1306                                 const unsigned last_bus)
1307 {
1308         struct i7core_dev *i7core_dev;
1309         const struct pci_id_descr *dev_descr = &table->descr[devno];
1310
1311         struct pci_dev *pdev = NULL;
1312         u8 bus = 0;
1313         u8 socket = 0;
1314
1315         pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1316                               dev_descr->dev_id, *prev);
1317
1318         /*
1319          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core regs
1320          * is at addr 8086:2c40, instead of 8086:2c41. So, we need
1321          * to probe for the alternate address in case of failure
1322          */
1323         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev)
1324                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1325                                       PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev);
1326
1327         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE && !pdev)
1328                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1329                                       PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT,
1330                                       *prev);
1331
1332         if (!pdev) {
1333                 if (*prev) {
1334                         *prev = pdev;
1335                         return 0;
1336                 }
1337
1338                 if (dev_descr->optional)
1339                         return 0;
1340
1341                 if (devno == 0)
1342                         return -ENODEV;
1343
1344                 i7core_printk(KERN_INFO,
1345                         "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1346                         dev_descr->dev, dev_descr->func,
1347                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1348
1349                 /* End of list, leave */
1350                 return -ENODEV;
1351         }
1352         bus = pdev->bus->number;
1353
1354         socket = last_bus - bus;
1355
1356         i7core_dev = get_i7core_dev(socket);
1357         if (!i7core_dev) {
1358                 i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
1359                 if (!i7core_dev)
1360                         return -ENOMEM;
1361                 i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev)
1362                                                 * table->n_devs, GFP_KERNEL);
1363                 if (!i7core_dev->pdev) {
1364                         kfree(i7core_dev);
1365                         return -ENOMEM;
1366                 }
1367                 i7core_dev->socket = socket;
1368                 i7core_dev->n_devs = table->n_devs;
1369                 list_add_tail(&i7core_dev->list, &i7core_edac_list);
1370         }
1371
1372         if (i7core_dev->pdev[devno]) {
1373                 i7core_printk(KERN_ERR,
1374                         "Duplicated device for "
1375                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1376                         bus, dev_descr->dev, dev_descr->func,
1377                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1378                 pci_dev_put(pdev);
1379                 return -ENODEV;
1380         }
1381
1382         i7core_dev->pdev[devno] = pdev;
1383
1384         /* Sanity check */
1385         if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1386                         PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1387                 i7core_printk(KERN_ERR,
1388                         "Device PCI ID %04x:%04x "
1389                         "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1390                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1391                         bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1392                         bus, dev_descr->dev, dev_descr->func);
1393                 return -ENODEV;
1394         }
1395
1396         /* Be sure that the device is enabled */
1397         if (unlikely(pci_enable_device(pdev) < 0)) {
1398                 i7core_printk(KERN_ERR,
1399                         "Couldn't enable "
1400                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1401                         bus, dev_descr->dev, dev_descr->func,
1402                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1403                 return -ENODEV;
1404         }
1405
1406         debugf0("Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1407                 socket, bus, dev_descr->dev,
1408                 dev_descr->func,
1409                 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1410
1411         *prev = pdev;
1412
1413         return 0;
1414 }
1415
1416 static int i7core_get_devices(const struct pci_id_table *table)
1417 {
1418         int i, rc, last_bus;
1419         struct pci_dev *pdev = NULL;
1420
1421         last_bus = i7core_pci_lastbus();
1422
1423         while (table && table->descr) {
1424                 for (i = 0; i < table->n_devs; i++) {
1425                         pdev = NULL;
1426                         do {
1427                                 rc = i7core_get_onedevice(&pdev, table, i,
1428                                                           last_bus);
1429                                 if (rc < 0) {
1430                                         if (i == 0) {
1431                                                 i = table->n_devs;
1432                                                 break;
1433                                         }
1434                                         i7core_put_all_devices();
1435                                         return -ENODEV;
1436                                 }
1437                         } while (pdev);
1438                 }
1439                 table++;
1440         }
1441
1442         return 0;
1443 }
1444
1445 static int mci_bind_devs(struct mem_ctl_info *mci,
1446                          struct i7core_dev *i7core_dev)
1447 {
1448         struct i7core_pvt *pvt = mci->pvt_info;
1449         struct pci_dev *pdev;
1450         int i, func, slot;
1451
1452         /* Associates i7core_dev and mci for future usage */
1453         pvt->i7core_dev = i7core_dev;
1454         i7core_dev->mci = mci;
1455
1456         pvt->is_registered = 0;
1457         for (i = 0; i < i7core_dev->n_devs; i++) {
1458                 pdev = i7core_dev->pdev[i];
1459                 if (!pdev)
1460                         continue;
1461
1462                 func = PCI_FUNC(pdev->devfn);
1463                 slot = PCI_SLOT(pdev->devfn);
1464                 if (slot == 3) {
1465                         if (unlikely(func > MAX_MCR_FUNC))
1466                                 goto error;
1467                         pvt->pci_mcr[func] = pdev;
1468                 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1469                         if (unlikely(func > MAX_CHAN_FUNC))
1470                                 goto error;
1471                         pvt->pci_ch[slot - 4][func] = pdev;
1472                 } else if (!slot && !func)
1473                         pvt->pci_noncore = pdev;
1474                 else
1475                         goto error;
1476
1477                 debugf0("Associated fn %d.%d, dev = %p, socket %d\n",
1478                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1479                         pdev, i7core_dev->socket);
1480
1481                 if (PCI_SLOT(pdev->devfn) == 3 &&
1482                         PCI_FUNC(pdev->devfn) == 2)
1483                         pvt->is_registered = 1;
1484         }
1485
1486         return 0;
1487
1488 error:
1489         i7core_printk(KERN_ERR, "Device %d, function %d "
1490                       "is out of the expected range\n",
1491                       slot, func);
1492         return -EINVAL;
1493 }
1494
1495 /****************************************************************************
1496                         Error check routines
1497  ****************************************************************************/
1498 static void i7core_rdimm_update_csrow(struct mem_ctl_info *mci,
1499                                       const int chan,
1500                                       const int dimm,
1501                                       const int add)
1502 {
1503         char *msg;
1504         struct i7core_pvt *pvt = mci->pvt_info;
1505         int row = pvt->csrow_map[chan][dimm], i;
1506
1507         for (i = 0; i < add; i++) {
1508                 msg = kasprintf(GFP_KERNEL, "Corrected error "
1509                                 "(Socket=%d channel=%d dimm=%d)",
1510                                 pvt->i7core_dev->socket, chan, dimm);
1511
1512                 edac_mc_handle_fbd_ce(mci, row, 0, msg);
1513                 kfree (msg);
1514         }
1515 }
1516
1517 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1518                                          const int chan,
1519                                          const int new0,
1520                                          const int new1,
1521                                          const int new2)
1522 {
1523         struct i7core_pvt *pvt = mci->pvt_info;
1524         int add0 = 0, add1 = 0, add2 = 0;
1525         /* Updates CE counters if it is not the first time here */
1526         if (pvt->ce_count_available) {
1527                 /* Updates CE counters */
1528
1529                 add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1530                 add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1531                 add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1532
1533                 if (add2 < 0)
1534                         add2 += 0x7fff;
1535                 pvt->rdimm_ce_count[chan][2] += add2;
1536
1537                 if (add1 < 0)
1538                         add1 += 0x7fff;
1539                 pvt->rdimm_ce_count[chan][1] += add1;
1540
1541                 if (add0 < 0)
1542                         add0 += 0x7fff;
1543                 pvt->rdimm_ce_count[chan][0] += add0;
1544         } else
1545                 pvt->ce_count_available = 1;
1546
1547         /* Store the new values */
1548         pvt->rdimm_last_ce_count[chan][2] = new2;
1549         pvt->rdimm_last_ce_count[chan][1] = new1;
1550         pvt->rdimm_last_ce_count[chan][0] = new0;
1551
1552         /*updated the edac core */
1553         if (add0 != 0)
1554                 i7core_rdimm_update_csrow(mci, chan, 0, add0);
1555         if (add1 != 0)
1556                 i7core_rdimm_update_csrow(mci, chan, 1, add1);
1557         if (add2 != 0)
1558                 i7core_rdimm_update_csrow(mci, chan, 2, add2);
1559
1560 }
1561
1562 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1563 {
1564         struct i7core_pvt *pvt = mci->pvt_info;
1565         u32 rcv[3][2];
1566         int i, new0, new1, new2;
1567
1568         /*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1569         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1570                                                                 &rcv[0][0]);
1571         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1572                                                                 &rcv[0][1]);
1573         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1574                                                                 &rcv[1][0]);
1575         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1576                                                                 &rcv[1][1]);
1577         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1578                                                                 &rcv[2][0]);
1579         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1580                                                                 &rcv[2][1]);
1581         for (i = 0 ; i < 3; i++) {
1582                 debugf3("MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1583                         (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1584                 /*if the channel has 3 dimms*/
1585                 if (pvt->channel[i].dimms > 2) {
1586                         new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1587                         new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1588                         new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1589                 } else {
1590                         new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1591                                         DIMM_BOT_COR_ERR(rcv[i][0]);
1592                         new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1593                                         DIMM_BOT_COR_ERR(rcv[i][1]);
1594                         new2 = 0;
1595                 }
1596
1597                 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1598         }
1599 }
1600
1601 /* This function is based on the device 3 function 4 registers as described on:
1602  * Intel Xeon Processor 5500 Series Datasheet Volume 2
1603  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1604  * also available at:
1605  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1606  */
1607 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1608 {
1609         struct i7core_pvt *pvt = mci->pvt_info;
1610         u32 rcv1, rcv0;
1611         int new0, new1, new2;
1612
1613         if (!pvt->pci_mcr[4]) {
1614                 debugf0("%s MCR registers not found\n", __func__);
1615                 return;
1616         }
1617
1618         /* Corrected test errors */
1619         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1620         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1621
1622         /* Store the new values */
1623         new2 = DIMM2_COR_ERR(rcv1);
1624         new1 = DIMM1_COR_ERR(rcv0);
1625         new0 = DIMM0_COR_ERR(rcv0);
1626
1627         /* Updates CE counters if it is not the first time here */
1628         if (pvt->ce_count_available) {
1629                 /* Updates CE counters */
1630                 int add0, add1, add2;
1631
1632                 add2 = new2 - pvt->udimm_last_ce_count[2];
1633                 add1 = new1 - pvt->udimm_last_ce_count[1];
1634                 add0 = new0 - pvt->udimm_last_ce_count[0];
1635
1636                 if (add2 < 0)
1637                         add2 += 0x7fff;
1638                 pvt->udimm_ce_count[2] += add2;
1639
1640                 if (add1 < 0)
1641                         add1 += 0x7fff;
1642                 pvt->udimm_ce_count[1] += add1;
1643
1644                 if (add0 < 0)
1645                         add0 += 0x7fff;
1646                 pvt->udimm_ce_count[0] += add0;
1647
1648                 if (add0 | add1 | add2)
1649                         i7core_printk(KERN_ERR, "New Corrected error(s): "
1650                                       "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1651                                       add0, add1, add2);
1652         } else
1653                 pvt->ce_count_available = 1;
1654
1655         /* Store the new values */
1656         pvt->udimm_last_ce_count[2] = new2;
1657         pvt->udimm_last_ce_count[1] = new1;
1658         pvt->udimm_last_ce_count[0] = new0;
1659 }
1660
1661 /*
1662  * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1663  * Architectures Software Developer’s Manual Volume 3B.
1664  * Nehalem are defined as family 0x06, model 0x1a
1665  *
1666  * The MCA registers used here are the following ones:
1667  *     struct mce field MCA Register
1668  *     m->status        MSR_IA32_MC8_STATUS
1669  *     m->addr          MSR_IA32_MC8_ADDR
1670  *     m->misc          MSR_IA32_MC8_MISC
1671  * In the case of Nehalem, the error information is masked at .status and .misc
1672  * fields
1673  */
1674 static void i7core_mce_output_error(struct mem_ctl_info *mci,
1675                                     const struct mce *m)
1676 {
1677         struct i7core_pvt *pvt = mci->pvt_info;
1678         char *type, *optype, *err, *msg;
1679         unsigned long error = m->status & 0x1ff0000l;
1680         u32 optypenum = (m->status >> 4) & 0x07;
1681         u32 core_err_cnt = (m->status >> 38) && 0x7fff;
1682         u32 dimm = (m->misc >> 16) & 0x3;
1683         u32 channel = (m->misc >> 18) & 0x3;
1684         u32 syndrome = m->misc >> 32;
1685         u32 errnum = find_first_bit(&error, 32);
1686         int csrow;
1687
1688         if (m->mcgstatus & 1)
1689                 type = "FATAL";
1690         else
1691                 type = "NON_FATAL";
1692
1693         switch (optypenum) {
1694         case 0:
1695                 optype = "generic undef request";
1696                 break;
1697         case 1:
1698                 optype = "read error";
1699                 break;
1700         case 2:
1701                 optype = "write error";
1702                 break;
1703         case 3:
1704                 optype = "addr/cmd error";
1705                 break;
1706         case 4:
1707                 optype = "scrubbing error";
1708                 break;
1709         default:
1710                 optype = "reserved";
1711                 break;
1712         }
1713
1714         switch (errnum) {
1715         case 16:
1716                 err = "read ECC error";
1717                 break;
1718         case 17:
1719                 err = "RAS ECC error";
1720                 break;
1721         case 18:
1722                 err = "write parity error";
1723                 break;
1724         case 19:
1725                 err = "redundacy loss";
1726                 break;
1727         case 20:
1728                 err = "reserved";
1729                 break;
1730         case 21:
1731                 err = "memory range error";
1732                 break;
1733         case 22:
1734                 err = "RTID out of range";
1735                 break;
1736         case 23:
1737                 err = "address parity error";
1738                 break;
1739         case 24:
1740                 err = "byte enable parity error";
1741                 break;
1742         default:
1743                 err = "unknown";
1744         }
1745
1746         /* FIXME: should convert addr into bank and rank information */
1747         msg = kasprintf(GFP_ATOMIC,
1748                 "%s (addr = 0x%08llx, cpu=%d, Dimm=%d, Channel=%d, "
1749                 "syndrome=0x%08x, count=%d, Err=%08llx:%08llx (%s: %s))\n",
1750                 type, (long long) m->addr, m->cpu, dimm, channel,
1751                 syndrome, core_err_cnt, (long long)m->status,
1752                 (long long)m->misc, optype, err);
1753
1754         debugf0("%s", msg);
1755
1756         csrow = pvt->csrow_map[channel][dimm];
1757
1758         /* Call the helper to output message */
1759         if (m->mcgstatus & 1)
1760                 edac_mc_handle_fbd_ue(mci, csrow, 0,
1761                                 0 /* FIXME: should be channel here */, msg);
1762         else if (!pvt->is_registered)
1763                 edac_mc_handle_fbd_ce(mci, csrow,
1764                                 0 /* FIXME: should be channel here */, msg);
1765
1766         kfree(msg);
1767 }
1768
1769 /*
1770  *      i7core_check_error      Retrieve and process errors reported by the
1771  *                              hardware. Called by the Core module.
1772  */
1773 static void i7core_check_error(struct mem_ctl_info *mci)
1774 {
1775         struct i7core_pvt *pvt = mci->pvt_info;
1776         int i;
1777         unsigned count = 0;
1778         struct mce *m;
1779
1780         /*
1781          * MCE first step: Copy all mce errors into a temporary buffer
1782          * We use a double buffering here, to reduce the risk of
1783          * loosing an error.
1784          */
1785         smp_rmb();
1786         count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1787                 % MCE_LOG_LEN;
1788         if (!count)
1789                 goto check_ce_error;
1790
1791         m = pvt->mce_outentry;
1792         if (pvt->mce_in + count > MCE_LOG_LEN) {
1793                 unsigned l = MCE_LOG_LEN - pvt->mce_in;
1794
1795                 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1796                 smp_wmb();
1797                 pvt->mce_in = 0;
1798                 count -= l;
1799                 m += l;
1800         }
1801         memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1802         smp_wmb();
1803         pvt->mce_in += count;
1804
1805         smp_rmb();
1806         if (pvt->mce_overrun) {
1807                 i7core_printk(KERN_ERR, "Lost %d memory errors\n",
1808                               pvt->mce_overrun);
1809                 smp_wmb();
1810                 pvt->mce_overrun = 0;
1811         }
1812
1813         /*
1814          * MCE second step: parse errors and display
1815          */
1816         for (i = 0; i < count; i++)
1817                 i7core_mce_output_error(mci, &pvt->mce_outentry[i]);
1818
1819         /*
1820          * Now, let's increment CE error counts
1821          */
1822 check_ce_error:
1823         if (!pvt->is_registered)
1824                 i7core_udimm_check_mc_ecc_err(mci);
1825         else
1826                 i7core_rdimm_check_mc_ecc_err(mci);
1827 }
1828
1829 /*
1830  * i7core_mce_check_error       Replicates mcelog routine to get errors
1831  *                              This routine simply queues mcelog errors, and
1832  *                              return. The error itself should be handled later
1833  *                              by i7core_check_error.
1834  * WARNING: As this routine should be called at NMI time, extra care should
1835  * be taken to avoid deadlocks, and to be as fast as possible.
1836  */
1837 static int i7core_mce_check_error(void *priv, struct mce *mce)
1838 {
1839         struct mem_ctl_info *mci = priv;
1840         struct i7core_pvt *pvt = mci->pvt_info;
1841
1842         /*
1843          * Just let mcelog handle it if the error is
1844          * outside the memory controller
1845          */
1846         if (((mce->status & 0xffff) >> 7) != 1)
1847                 return 0;
1848
1849         /* Bank 8 registers are the only ones that we know how to handle */
1850         if (mce->bank != 8)
1851                 return 0;
1852
1853 #ifdef CONFIG_SMP
1854         /* Only handle if it is the right mc controller */
1855         if (cpu_data(mce->cpu).phys_proc_id != pvt->i7core_dev->socket)
1856                 return 0;
1857 #endif
1858
1859         smp_rmb();
1860         if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1861                 smp_wmb();
1862                 pvt->mce_overrun++;
1863                 return 0;
1864         }
1865
1866         /* Copy memory error at the ringbuffer */
1867         memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1868         smp_wmb();
1869         pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1870
1871         /* Handle fatal errors immediately */
1872         if (mce->mcgstatus & 1)
1873                 i7core_check_error(mci);
1874
1875         /* Advice mcelog that the error were handled */
1876         return 1;
1877 }
1878
1879 static int i7core_register_mci(struct i7core_dev *i7core_dev,
1880                                const int num_channels, const int num_csrows)
1881 {
1882         struct mem_ctl_info *mci;
1883         struct i7core_pvt *pvt;
1884         int csrow = 0;
1885         int rc;
1886
1887         /* allocate a new MC control structure */
1888         mci = edac_mc_alloc(sizeof(*pvt), num_csrows, num_channels,
1889                             i7core_dev->socket);
1890         if (unlikely(!mci))
1891                 return -ENOMEM;
1892
1893         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
1894                 __func__, mci, &i7core_dev->pdev[0]->dev);
1895
1896         /* record ptr to the generic device */
1897         mci->dev = &i7core_dev->pdev[0]->dev;
1898
1899         pvt = mci->pvt_info;
1900         memset(pvt, 0, sizeof(*pvt));
1901
1902         /*
1903          * FIXME: how to handle RDDR3 at MCI level? It is possible to have
1904          * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
1905          * memory channels
1906          */
1907         mci->mtype_cap = MEM_FLAG_DDR3;
1908         mci->edac_ctl_cap = EDAC_FLAG_NONE;
1909         mci->edac_cap = EDAC_FLAG_NONE;
1910         mci->mod_name = "i7core_edac.c";
1911         mci->mod_ver = I7CORE_REVISION;
1912         mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
1913                                   i7core_dev->socket);
1914         mci->dev_name = pci_name(i7core_dev->pdev[0]);
1915         mci->ctl_page_to_phys = NULL;
1916
1917         if (pvt->is_registered)
1918                 mci->mc_driver_sysfs_attributes = i7core_sysfs_rdimm_attrs;
1919         else
1920                 mci->mc_driver_sysfs_attributes = i7core_sysfs_udimm_attrs;
1921
1922         /* Set the function pointer to an actual operation function */
1923         mci->edac_check = i7core_check_error;
1924
1925         /* Store pci devices at mci for faster access */
1926         rc = mci_bind_devs(mci, i7core_dev);
1927         if (unlikely(rc < 0))
1928                 goto fail;
1929
1930         /* Get dimm basic config */
1931         get_dimm_config(mci, &csrow);
1932
1933         /* add this new MC control structure to EDAC's list of MCs */
1934         if (unlikely(edac_mc_add_mc(mci))) {
1935                 debugf0("MC: " __FILE__
1936                         ": %s(): failed edac_mc_add_mc()\n", __func__);
1937                 /* FIXME: perhaps some code should go here that disables error
1938                  * reporting if we just enabled it
1939                  */
1940
1941                 rc = -EINVAL;
1942                 goto fail;
1943         }
1944
1945         /* Default error mask is any memory */
1946         pvt->inject.channel = 0;
1947         pvt->inject.dimm = -1;
1948         pvt->inject.rank = -1;
1949         pvt->inject.bank = -1;
1950         pvt->inject.page = -1;
1951         pvt->inject.col = -1;
1952
1953         /* Registers on edac_mce in order to receive memory errors */
1954         pvt->edac_mce.priv = mci;
1955         pvt->edac_mce.check_error = i7core_mce_check_error;
1956
1957         /* allocating generic PCI control info */
1958         pvt->i7core_pci = edac_pci_create_generic_ctl(&i7core_dev->pdev[0]->dev,
1959                                                  EDAC_MOD_STR);
1960         if (unlikely(!pvt->i7core_pci)) {
1961                 printk(KERN_WARNING
1962                         "%s(): Unable to create PCI control\n",
1963                         __func__);
1964                 printk(KERN_WARNING
1965                         "%s(): PCI error report via EDAC not setup\n",
1966                         __func__);
1967         }
1968
1969         rc = edac_mce_register(&pvt->edac_mce);
1970         if (unlikely(rc < 0)) {
1971                 debugf0("MC: " __FILE__
1972                         ": %s(): failed edac_mce_register()\n", __func__);
1973         }
1974
1975 fail:
1976         if (rc < 0)
1977                 edac_mc_free(mci);
1978         return rc;
1979 }
1980
1981 /*
1982  *      i7core_probe    Probe for ONE instance of device to see if it is
1983  *                      present.
1984  *      return:
1985  *              0 for FOUND a device
1986  *              < 0 for error code
1987  */
1988
1989 static int __devinit i7core_probe(struct pci_dev *pdev,
1990                                   const struct pci_device_id *id)
1991 {
1992         int rc;
1993         struct i7core_dev *i7core_dev;
1994
1995         /* get the pci devices we want to reserve for our use */
1996         mutex_lock(&i7core_edac_lock);
1997
1998         /*
1999          * All memory controllers are allocated at the first pass.
2000          */
2001         if (unlikely(probed >= 1)) {
2002                 mutex_unlock(&i7core_edac_lock);
2003                 return -EINVAL;
2004         }
2005         probed++;
2006
2007         rc = i7core_get_devices(pci_dev_table);
2008         if (unlikely(rc < 0))
2009                 goto fail0;
2010
2011         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2012                 int channels;
2013                 int csrows;
2014
2015                 /* Check the number of active and not disabled channels */
2016                 rc = i7core_get_active_channels(i7core_dev->socket,
2017                                                 &channels, &csrows);
2018                 if (unlikely(rc < 0))
2019                         goto fail1;
2020
2021                 rc = i7core_register_mci(i7core_dev, channels, csrows);
2022                 if (unlikely(rc < 0))
2023                         goto fail1;
2024         }
2025
2026         i7core_printk(KERN_INFO, "Driver loaded.\n");
2027
2028         mutex_unlock(&i7core_edac_lock);
2029         return 0;
2030
2031 fail1:
2032         i7core_put_all_devices();
2033 fail0:
2034         mutex_unlock(&i7core_edac_lock);
2035         return rc;
2036 }
2037
2038 /*
2039  *      i7core_remove   destructor for one instance of device
2040  *
2041  */
2042 static void __devexit i7core_remove(struct pci_dev *pdev)
2043 {
2044         struct mem_ctl_info *mci;
2045         struct i7core_dev *i7core_dev, *tmp;
2046         struct i7core_pvt *pvt;
2047
2048         debugf0(__FILE__ ": %s()\n", __func__);
2049
2050         /*
2051          * we have a trouble here: pdev value for removal will be wrong, since
2052          * it will point to the X58 register used to detect that the machine
2053          * is a Nehalem or upper design. However, due to the way several PCI
2054          * devices are grouped together to provide MC functionality, we need
2055          * to use a different method for releasing the devices
2056          */
2057
2058         mutex_lock(&i7core_edac_lock);
2059         list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
2060                 mci = find_mci_by_dev(&i7core_dev->pdev[0]->dev);
2061                 if (unlikely(!mci || !mci->pvt_info)) {
2062                         debugf0("MC: " __FILE__ ": %s(): dev = %p\n",
2063                                 __func__, &i7core_dev->pdev[0]->dev);
2064
2065                                 i7core_printk(KERN_ERR,
2066                                       "Couldn't find mci hanler\n");
2067                 } else {
2068                         pvt = mci->pvt_info;
2069                         i7core_dev = pvt->i7core_dev;
2070
2071                         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2072                                 __func__, mci, &i7core_dev->pdev[0]->dev);
2073
2074                         /* Disable MCE NMI handler */
2075                         edac_mce_unregister(&pvt->edac_mce);
2076
2077                         /* Disable EDAC polling */
2078                         if (likely(pvt->i7core_pci))
2079                                 edac_pci_release_generic_ctl(pvt->i7core_pci);
2080                         else
2081                                 i7core_printk(KERN_ERR,
2082                                               "Couldn't find mem_ctl_info for socket %d\n",
2083                                               i7core_dev->socket);
2084                         pvt->i7core_pci = NULL;
2085
2086                         /* Remove MC sysfs nodes */
2087                         edac_mc_del_mc(&i7core_dev->pdev[0]->dev);
2088
2089                         debugf1("%s: free mci struct\n", mci->ctl_name);
2090                         kfree(mci->ctl_name);
2091                         edac_mc_free(mci);
2092
2093                         /* Release PCI resources */
2094                         i7core_put_devices(i7core_dev);
2095                         list_del(&i7core_dev->list);
2096                         kfree(i7core_dev);
2097                 }
2098         }
2099         probed--;
2100
2101         mutex_unlock(&i7core_edac_lock);
2102 }
2103
2104 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2105
2106 /*
2107  *      i7core_driver   pci_driver structure for this module
2108  *
2109  */
2110 static struct pci_driver i7core_driver = {
2111         .name     = "i7core_edac",
2112         .probe    = i7core_probe,
2113         .remove   = __devexit_p(i7core_remove),
2114         .id_table = i7core_pci_tbl,
2115 };
2116
2117 /*
2118  *      i7core_init             Module entry function
2119  *                      Try to initialize this module for its devices
2120  */
2121 static int __init i7core_init(void)
2122 {
2123         int pci_rc;
2124
2125         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2126
2127         /* Ensure that the OPSTATE is set correctly for POLL or NMI */
2128         opstate_init();
2129
2130         if (use_pci_fixup)
2131                 i7core_xeon_pci_fixup(pci_dev_table);
2132
2133         pci_rc = pci_register_driver(&i7core_driver);
2134
2135         if (pci_rc >= 0)
2136                 return 0;
2137
2138         i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2139                       pci_rc);
2140
2141         return pci_rc;
2142 }
2143
2144 /*
2145  *      i7core_exit()   Module exit function
2146  *                      Unregister the driver
2147  */
2148 static void __exit i7core_exit(void)
2149 {
2150         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2151         pci_unregister_driver(&i7core_driver);
2152 }
2153
2154 module_init(i7core_init);
2155 module_exit(i7core_exit);
2156
2157 MODULE_LICENSE("GPL");
2158 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
2159 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
2160 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2161                    I7CORE_REVISION);
2162
2163 module_param(edac_op_state, int, 0444);
2164 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");