i7core_edac: Don't do the legacy PCI probe by default
[pandora-kernel.git] / drivers / edac / i7core_edac.c
1 /* Intel i7 core/Nehalem Memory Controller kernel module
2  *
3  * This driver supports yhe memory controllers found on the Intel
4  * processor families i7core, i7core 7xx/8xx, i5core, Xeon 35xx,
5  * Xeon 55xx and Xeon 56xx also known as Nehalem, Nehalem-EP, Lynnfield
6  * and Westmere-EP.
7  *
8  * This file may be distributed under the terms of the
9  * GNU General Public License version 2 only.
10  *
11  * Copyright (c) 2009-2010 by:
12  *       Mauro Carvalho Chehab <mchehab@redhat.com>
13  *
14  * Red Hat Inc. http://www.redhat.com
15  *
16  * Forked and adapted from the i5400_edac driver
17  *
18  * Based on the following public Intel datasheets:
19  * Intel Core i7 Processor Extreme Edition and Intel Core i7 Processor
20  * Datasheet, Volume 2:
21  *      http://download.intel.com/design/processor/datashts/320835.pdf
22  * Intel Xeon Processor 5500 Series Datasheet Volume 2
23  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
24  * also available at:
25  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
26  */
27
28 #include <linux/module.h>
29 #include <linux/init.h>
30 #include <linux/pci.h>
31 #include <linux/pci_ids.h>
32 #include <linux/slab.h>
33 #include <linux/delay.h>
34 #include <linux/edac.h>
35 #include <linux/mmzone.h>
36 #include <linux/edac_mce.h>
37 #include <linux/smp.h>
38 #include <asm/processor.h>
39
40 #include "edac_core.h"
41
42 /* Static vars */
43 static LIST_HEAD(i7core_edac_list);
44 static DEFINE_MUTEX(i7core_edac_lock);
45 static int probed;
46
47 static int use_pci_fixup;
48 module_param(use_pci_fixup, int, 0444);
49 MODULE_PARM_DESC(use_pci_fixup, "Enable PCI fixup to seek for hidden devices");
50 /*
51  * This is used for Nehalem-EP and Nehalem-EX devices, where the non-core
52  * registers start at bus 255, and are not reported by BIOS.
53  * We currently find devices with only 2 sockets. In order to support more QPI
54  * Quick Path Interconnect, just increment this number.
55  */
56 #define MAX_SOCKET_BUSES        2
57
58
59 /*
60  * Alter this version for the module when modifications are made
61  */
62 #define I7CORE_REVISION    " Ver: 1.0.0 " __DATE__
63 #define EDAC_MOD_STR      "i7core_edac"
64
65 /*
66  * Debug macros
67  */
68 #define i7core_printk(level, fmt, arg...)                       \
69         edac_printk(level, "i7core", fmt, ##arg)
70
71 #define i7core_mc_printk(mci, level, fmt, arg...)               \
72         edac_mc_chipset_printk(mci, level, "i7core", fmt, ##arg)
73
74 /*
75  * i7core Memory Controller Registers
76  */
77
78         /* OFFSETS for Device 0 Function 0 */
79
80 #define MC_CFG_CONTROL  0x90
81
82         /* OFFSETS for Device 3 Function 0 */
83
84 #define MC_CONTROL      0x48
85 #define MC_STATUS       0x4c
86 #define MC_MAX_DOD      0x64
87
88 /*
89  * OFFSETS for Device 3 Function 4, as inicated on Xeon 5500 datasheet:
90  * http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
91  */
92
93 #define MC_TEST_ERR_RCV1        0x60
94   #define DIMM2_COR_ERR(r)                      ((r) & 0x7fff)
95
96 #define MC_TEST_ERR_RCV0        0x64
97   #define DIMM1_COR_ERR(r)                      (((r) >> 16) & 0x7fff)
98   #define DIMM0_COR_ERR(r)                      ((r) & 0x7fff)
99
100 /* OFFSETS for Device 3 Function 2, as inicated on Xeon 5500 datasheet */
101 #define MC_COR_ECC_CNT_0        0x80
102 #define MC_COR_ECC_CNT_1        0x84
103 #define MC_COR_ECC_CNT_2        0x88
104 #define MC_COR_ECC_CNT_3        0x8c
105 #define MC_COR_ECC_CNT_4        0x90
106 #define MC_COR_ECC_CNT_5        0x94
107
108 #define DIMM_TOP_COR_ERR(r)                     (((r) >> 16) & 0x7fff)
109 #define DIMM_BOT_COR_ERR(r)                     ((r) & 0x7fff)
110
111
112         /* OFFSETS for Devices 4,5 and 6 Function 0 */
113
114 #define MC_CHANNEL_DIMM_INIT_PARAMS 0x58
115   #define THREE_DIMMS_PRESENT           (1 << 24)
116   #define SINGLE_QUAD_RANK_PRESENT      (1 << 23)
117   #define QUAD_RANK_PRESENT             (1 << 22)
118   #define REGISTERED_DIMM               (1 << 15)
119
120 #define MC_CHANNEL_MAPPER       0x60
121   #define RDLCH(r, ch)          ((((r) >> (3 + (ch * 6))) & 0x07) - 1)
122   #define WRLCH(r, ch)          ((((r) >> (ch * 6)) & 0x07) - 1)
123
124 #define MC_CHANNEL_RANK_PRESENT 0x7c
125   #define RANK_PRESENT_MASK             0xffff
126
127 #define MC_CHANNEL_ADDR_MATCH   0xf0
128 #define MC_CHANNEL_ERROR_MASK   0xf8
129 #define MC_CHANNEL_ERROR_INJECT 0xfc
130   #define INJECT_ADDR_PARITY    0x10
131   #define INJECT_ECC            0x08
132   #define MASK_CACHELINE        0x06
133   #define MASK_FULL_CACHELINE   0x06
134   #define MASK_MSB32_CACHELINE  0x04
135   #define MASK_LSB32_CACHELINE  0x02
136   #define NO_MASK_CACHELINE     0x00
137   #define REPEAT_EN             0x01
138
139         /* OFFSETS for Devices 4,5 and 6 Function 1 */
140
141 #define MC_DOD_CH_DIMM0         0x48
142 #define MC_DOD_CH_DIMM1         0x4c
143 #define MC_DOD_CH_DIMM2         0x50
144   #define RANKOFFSET_MASK       ((1 << 12) | (1 << 11) | (1 << 10))
145   #define RANKOFFSET(x)         ((x & RANKOFFSET_MASK) >> 10)
146   #define DIMM_PRESENT_MASK     (1 << 9)
147   #define DIMM_PRESENT(x)       (((x) & DIMM_PRESENT_MASK) >> 9)
148   #define MC_DOD_NUMBANK_MASK           ((1 << 8) | (1 << 7))
149   #define MC_DOD_NUMBANK(x)             (((x) & MC_DOD_NUMBANK_MASK) >> 7)
150   #define MC_DOD_NUMRANK_MASK           ((1 << 6) | (1 << 5))
151   #define MC_DOD_NUMRANK(x)             (((x) & MC_DOD_NUMRANK_MASK) >> 5)
152   #define MC_DOD_NUMROW_MASK            ((1 << 4) | (1 << 3) | (1 << 2))
153   #define MC_DOD_NUMROW(x)              (((x) & MC_DOD_NUMROW_MASK) >> 2)
154   #define MC_DOD_NUMCOL_MASK            3
155   #define MC_DOD_NUMCOL(x)              ((x) & MC_DOD_NUMCOL_MASK)
156
157 #define MC_RANK_PRESENT         0x7c
158
159 #define MC_SAG_CH_0     0x80
160 #define MC_SAG_CH_1     0x84
161 #define MC_SAG_CH_2     0x88
162 #define MC_SAG_CH_3     0x8c
163 #define MC_SAG_CH_4     0x90
164 #define MC_SAG_CH_5     0x94
165 #define MC_SAG_CH_6     0x98
166 #define MC_SAG_CH_7     0x9c
167
168 #define MC_RIR_LIMIT_CH_0       0x40
169 #define MC_RIR_LIMIT_CH_1       0x44
170 #define MC_RIR_LIMIT_CH_2       0x48
171 #define MC_RIR_LIMIT_CH_3       0x4C
172 #define MC_RIR_LIMIT_CH_4       0x50
173 #define MC_RIR_LIMIT_CH_5       0x54
174 #define MC_RIR_LIMIT_CH_6       0x58
175 #define MC_RIR_LIMIT_CH_7       0x5C
176 #define MC_RIR_LIMIT_MASK       ((1 << 10) - 1)
177
178 #define MC_RIR_WAY_CH           0x80
179   #define MC_RIR_WAY_OFFSET_MASK        (((1 << 14) - 1) & ~0x7)
180   #define MC_RIR_WAY_RANK_MASK          0x7
181
182 /*
183  * i7core structs
184  */
185
186 #define NUM_CHANS 3
187 #define MAX_DIMMS 3             /* Max DIMMS per channel */
188 #define MAX_MCR_FUNC  4
189 #define MAX_CHAN_FUNC 3
190
191 struct i7core_info {
192         u32     mc_control;
193         u32     mc_status;
194         u32     max_dod;
195         u32     ch_map;
196 };
197
198
199 struct i7core_inject {
200         int     enable;
201
202         u32     section;
203         u32     type;
204         u32     eccmask;
205
206         /* Error address mask */
207         int channel, dimm, rank, bank, page, col;
208 };
209
210 struct i7core_channel {
211         u32             ranks;
212         u32             dimms;
213 };
214
215 struct pci_id_descr {
216         int                     dev;
217         int                     func;
218         int                     dev_id;
219         int                     optional;
220 };
221
222 struct pci_id_table {
223         const struct pci_id_descr       *descr;
224         int                             n_devs;
225 };
226
227 struct i7core_dev {
228         struct list_head        list;
229         u8                      socket;
230         struct pci_dev          **pdev;
231         int                     n_devs;
232         struct mem_ctl_info     *mci;
233 };
234
235 struct i7core_pvt {
236         struct pci_dev  *pci_noncore;
237         struct pci_dev  *pci_mcr[MAX_MCR_FUNC + 1];
238         struct pci_dev  *pci_ch[NUM_CHANS][MAX_CHAN_FUNC + 1];
239
240         struct i7core_dev *i7core_dev;
241
242         struct i7core_info      info;
243         struct i7core_inject    inject;
244         struct i7core_channel   channel[NUM_CHANS];
245
246         int             channels; /* Number of active channels */
247
248         int             ce_count_available;
249         int             csrow_map[NUM_CHANS][MAX_DIMMS];
250
251                         /* ECC corrected errors counts per udimm */
252         unsigned long   udimm_ce_count[MAX_DIMMS];
253         int             udimm_last_ce_count[MAX_DIMMS];
254                         /* ECC corrected errors counts per rdimm */
255         unsigned long   rdimm_ce_count[NUM_CHANS][MAX_DIMMS];
256         int             rdimm_last_ce_count[NUM_CHANS][MAX_DIMMS];
257
258         unsigned int    is_registered;
259
260         /* mcelog glue */
261         struct edac_mce         edac_mce;
262
263         /* Fifo double buffers */
264         struct mce              mce_entry[MCE_LOG_LEN];
265         struct mce              mce_outentry[MCE_LOG_LEN];
266
267         /* Fifo in/out counters */
268         unsigned                mce_in, mce_out;
269
270         /* Count indicator to show errors not got */
271         unsigned                mce_overrun;
272
273         /* Struct to control EDAC polling */
274         struct edac_pci_ctl_info *i7core_pci;
275 };
276
277 #define PCI_DESCR(device, function, device_id)  \
278         .dev = (device),                        \
279         .func = (function),                     \
280         .dev_id = (device_id)
281
282 static const struct pci_id_descr pci_dev_descr_i7core_nehalem[] = {
283                 /* Memory controller */
284         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_I7_MCR)     },
285         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_I7_MC_TAD)  },
286                         /* Exists only for RDIMM */
287         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_I7_MC_RAS), .optional = 1  },
288         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_I7_MC_TEST) },
289
290                 /* Channel 0 */
291         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH0_CTRL) },
292         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH0_ADDR) },
293         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH0_RANK) },
294         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH0_TC)   },
295
296                 /* Channel 1 */
297         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH1_CTRL) },
298         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH1_ADDR) },
299         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH1_RANK) },
300         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH1_TC)   },
301
302                 /* Channel 2 */
303         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_I7_MC_CH2_CTRL) },
304         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_I7_MC_CH2_ADDR) },
305         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_I7_MC_CH2_RANK) },
306         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_I7_MC_CH2_TC)   },
307
308                 /* Generic Non-core registers */
309         /*
310          * This is the PCI device on i7core and on Xeon 35xx (8086:2c41)
311          * On Xeon 55xx, however, it has a different id (8086:2c40). So,
312          * the probing code needs to test for the other address in case of
313          * failure of this one
314          */
315         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_I7_NONCORE)  },
316
317 };
318
319 static const struct pci_id_descr pci_dev_descr_lynnfield[] = {
320         { PCI_DESCR( 3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR)         },
321         { PCI_DESCR( 3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD)      },
322         { PCI_DESCR( 3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST)     },
323
324         { PCI_DESCR( 4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL) },
325         { PCI_DESCR( 4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR) },
326         { PCI_DESCR( 4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK) },
327         { PCI_DESCR( 4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC)   },
328
329         { PCI_DESCR( 5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL) },
330         { PCI_DESCR( 5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR) },
331         { PCI_DESCR( 5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK) },
332         { PCI_DESCR( 5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC)   },
333
334         /*
335          * This is the PCI device has an alternate address on some
336          * processors like Core i7 860
337          */
338         { PCI_DESCR( 0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE)     },
339 };
340
341 static const struct pci_id_descr pci_dev_descr_i7core_westmere[] = {
342                 /* Memory controller */
343         { PCI_DESCR(3, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MCR_REV2)     },
344         { PCI_DESCR(3, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TAD_REV2)  },
345                         /* Exists only for RDIMM */
346         { PCI_DESCR(3, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_RAS_REV2), .optional = 1  },
347         { PCI_DESCR(3, 4, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_TEST_REV2) },
348
349                 /* Channel 0 */
350         { PCI_DESCR(4, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_CTRL_REV2) },
351         { PCI_DESCR(4, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_ADDR_REV2) },
352         { PCI_DESCR(4, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_RANK_REV2) },
353         { PCI_DESCR(4, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH0_TC_REV2)   },
354
355                 /* Channel 1 */
356         { PCI_DESCR(5, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_CTRL_REV2) },
357         { PCI_DESCR(5, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_ADDR_REV2) },
358         { PCI_DESCR(5, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_RANK_REV2) },
359         { PCI_DESCR(5, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH1_TC_REV2)   },
360
361                 /* Channel 2 */
362         { PCI_DESCR(6, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_CTRL_REV2) },
363         { PCI_DESCR(6, 1, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_ADDR_REV2) },
364         { PCI_DESCR(6, 2, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_RANK_REV2) },
365         { PCI_DESCR(6, 3, PCI_DEVICE_ID_INTEL_LYNNFIELD_MC_CH2_TC_REV2)   },
366
367                 /* Generic Non-core registers */
368         { PCI_DESCR(0, 0, PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_REV2)  },
369
370 };
371
372 #define PCI_ID_TABLE_ENTRY(A) { .descr=A, .n_devs = ARRAY_SIZE(A) }
373 static const struct pci_id_table pci_dev_table[] = {
374         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_nehalem),
375         PCI_ID_TABLE_ENTRY(pci_dev_descr_lynnfield),
376         PCI_ID_TABLE_ENTRY(pci_dev_descr_i7core_westmere),
377 };
378
379 /*
380  *      pci_device_id   table for which devices we are looking for
381  */
382 static const struct pci_device_id i7core_pci_tbl[] __devinitdata = {
383         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_X58_HUB_MGMT)},
384         {PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_LYNNFIELD_QPI_LINK0)},
385         {0,}                    /* 0 terminated list. */
386 };
387
388 /****************************************************************************
389                         Anciliary status routines
390  ****************************************************************************/
391
392         /* MC_CONTROL bits */
393 #define CH_ACTIVE(pvt, ch)      ((pvt)->info.mc_control & (1 << (8 + ch)))
394 #define ECCx8(pvt)              ((pvt)->info.mc_control & (1 << 1))
395
396         /* MC_STATUS bits */
397 #define ECC_ENABLED(pvt)        ((pvt)->info.mc_status & (1 << 4))
398 #define CH_DISABLED(pvt, ch)    ((pvt)->info.mc_status & (1 << ch))
399
400         /* MC_MAX_DOD read functions */
401 static inline int numdimms(u32 dimms)
402 {
403         return (dimms & 0x3) + 1;
404 }
405
406 static inline int numrank(u32 rank)
407 {
408         static int ranks[4] = { 1, 2, 4, -EINVAL };
409
410         return ranks[rank & 0x3];
411 }
412
413 static inline int numbank(u32 bank)
414 {
415         static int banks[4] = { 4, 8, 16, -EINVAL };
416
417         return banks[bank & 0x3];
418 }
419
420 static inline int numrow(u32 row)
421 {
422         static int rows[8] = {
423                 1 << 12, 1 << 13, 1 << 14, 1 << 15,
424                 1 << 16, -EINVAL, -EINVAL, -EINVAL,
425         };
426
427         return rows[row & 0x7];
428 }
429
430 static inline int numcol(u32 col)
431 {
432         static int cols[8] = {
433                 1 << 10, 1 << 11, 1 << 12, -EINVAL,
434         };
435         return cols[col & 0x3];
436 }
437
438 static struct i7core_dev *get_i7core_dev(u8 socket)
439 {
440         struct i7core_dev *i7core_dev;
441
442         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
443                 if (i7core_dev->socket == socket)
444                         return i7core_dev;
445         }
446
447         return NULL;
448 }
449
450 /****************************************************************************
451                         Memory check routines
452  ****************************************************************************/
453 static struct pci_dev *get_pdev_slot_func(u8 socket, unsigned slot,
454                                           unsigned func)
455 {
456         struct i7core_dev *i7core_dev = get_i7core_dev(socket);
457         int i;
458
459         if (!i7core_dev)
460                 return NULL;
461
462         for (i = 0; i < i7core_dev->n_devs; i++) {
463                 if (!i7core_dev->pdev[i])
464                         continue;
465
466                 if (PCI_SLOT(i7core_dev->pdev[i]->devfn) == slot &&
467                     PCI_FUNC(i7core_dev->pdev[i]->devfn) == func) {
468                         return i7core_dev->pdev[i];
469                 }
470         }
471
472         return NULL;
473 }
474
475 /**
476  * i7core_get_active_channels() - gets the number of channels and csrows
477  * @socket:     Quick Path Interconnect socket
478  * @channels:   Number of channels that will be returned
479  * @csrows:     Number of csrows found
480  *
481  * Since EDAC core needs to know in advance the number of available channels
482  * and csrows, in order to allocate memory for csrows/channels, it is needed
483  * to run two similar steps. At the first step, implemented on this function,
484  * it checks the number of csrows/channels present at one socket.
485  * this is used in order to properly allocate the size of mci components.
486  *
487  * It should be noticed that none of the current available datasheets explain
488  * or even mention how csrows are seen by the memory controller. So, we need
489  * to add a fake description for csrows.
490  * So, this driver is attributing one DIMM memory for one csrow.
491  */
492 static int i7core_get_active_channels(const u8 socket, unsigned *channels,
493                                       unsigned *csrows)
494 {
495         struct pci_dev *pdev = NULL;
496         int i, j;
497         u32 status, control;
498
499         *channels = 0;
500         *csrows = 0;
501
502         pdev = get_pdev_slot_func(socket, 3, 0);
503         if (!pdev) {
504                 i7core_printk(KERN_ERR, "Couldn't find socket %d fn 3.0!!!\n",
505                               socket);
506                 return -ENODEV;
507         }
508
509         /* Device 3 function 0 reads */
510         pci_read_config_dword(pdev, MC_STATUS, &status);
511         pci_read_config_dword(pdev, MC_CONTROL, &control);
512
513         for (i = 0; i < NUM_CHANS; i++) {
514                 u32 dimm_dod[3];
515                 /* Check if the channel is active */
516                 if (!(control & (1 << (8 + i))))
517                         continue;
518
519                 /* Check if the channel is disabled */
520                 if (status & (1 << i))
521                         continue;
522
523                 pdev = get_pdev_slot_func(socket, i + 4, 1);
524                 if (!pdev) {
525                         i7core_printk(KERN_ERR, "Couldn't find socket %d "
526                                                 "fn %d.%d!!!\n",
527                                                 socket, i + 4, 1);
528                         return -ENODEV;
529                 }
530                 /* Devices 4-6 function 1 */
531                 pci_read_config_dword(pdev,
532                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
533                 pci_read_config_dword(pdev,
534                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
535                 pci_read_config_dword(pdev,
536                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
537
538                 (*channels)++;
539
540                 for (j = 0; j < 3; j++) {
541                         if (!DIMM_PRESENT(dimm_dod[j]))
542                                 continue;
543                         (*csrows)++;
544                 }
545         }
546
547         debugf0("Number of active channels on socket %d: %d\n",
548                 socket, *channels);
549
550         return 0;
551 }
552
553 static int get_dimm_config(const struct mem_ctl_info *mci, int *csrow)
554 {
555         struct i7core_pvt *pvt = mci->pvt_info;
556         struct csrow_info *csr;
557         struct pci_dev *pdev;
558         int i, j;
559         unsigned long last_page = 0;
560         enum edac_type mode;
561         enum mem_type mtype;
562
563         /* Get data from the MC register, function 0 */
564         pdev = pvt->pci_mcr[0];
565         if (!pdev)
566                 return -ENODEV;
567
568         /* Device 3 function 0 reads */
569         pci_read_config_dword(pdev, MC_CONTROL, &pvt->info.mc_control);
570         pci_read_config_dword(pdev, MC_STATUS, &pvt->info.mc_status);
571         pci_read_config_dword(pdev, MC_MAX_DOD, &pvt->info.max_dod);
572         pci_read_config_dword(pdev, MC_CHANNEL_MAPPER, &pvt->info.ch_map);
573
574         debugf0("QPI %d control=0x%08x status=0x%08x dod=0x%08x map=0x%08x\n",
575                 pvt->i7core_dev->socket, pvt->info.mc_control, pvt->info.mc_status,
576                 pvt->info.max_dod, pvt->info.ch_map);
577
578         if (ECC_ENABLED(pvt)) {
579                 debugf0("ECC enabled with x%d SDCC\n", ECCx8(pvt) ? 8 : 4);
580                 if (ECCx8(pvt))
581                         mode = EDAC_S8ECD8ED;
582                 else
583                         mode = EDAC_S4ECD4ED;
584         } else {
585                 debugf0("ECC disabled\n");
586                 mode = EDAC_NONE;
587         }
588
589         /* FIXME: need to handle the error codes */
590         debugf0("DOD Max limits: DIMMS: %d, %d-ranked, %d-banked "
591                 "x%x x 0x%x\n",
592                 numdimms(pvt->info.max_dod),
593                 numrank(pvt->info.max_dod >> 2),
594                 numbank(pvt->info.max_dod >> 4),
595                 numrow(pvt->info.max_dod >> 6),
596                 numcol(pvt->info.max_dod >> 9));
597
598         for (i = 0; i < NUM_CHANS; i++) {
599                 u32 data, dimm_dod[3], value[8];
600
601                 if (!pvt->pci_ch[i][0])
602                         continue;
603
604                 if (!CH_ACTIVE(pvt, i)) {
605                         debugf0("Channel %i is not active\n", i);
606                         continue;
607                 }
608                 if (CH_DISABLED(pvt, i)) {
609                         debugf0("Channel %i is disabled\n", i);
610                         continue;
611                 }
612
613                 /* Devices 4-6 function 0 */
614                 pci_read_config_dword(pvt->pci_ch[i][0],
615                                 MC_CHANNEL_DIMM_INIT_PARAMS, &data);
616
617                 pvt->channel[i].ranks = (data & QUAD_RANK_PRESENT) ?
618                                                 4 : 2;
619
620                 if (data & REGISTERED_DIMM)
621                         mtype = MEM_RDDR3;
622                 else
623                         mtype = MEM_DDR3;
624 #if 0
625                 if (data & THREE_DIMMS_PRESENT)
626                         pvt->channel[i].dimms = 3;
627                 else if (data & SINGLE_QUAD_RANK_PRESENT)
628                         pvt->channel[i].dimms = 1;
629                 else
630                         pvt->channel[i].dimms = 2;
631 #endif
632
633                 /* Devices 4-6 function 1 */
634                 pci_read_config_dword(pvt->pci_ch[i][1],
635                                 MC_DOD_CH_DIMM0, &dimm_dod[0]);
636                 pci_read_config_dword(pvt->pci_ch[i][1],
637                                 MC_DOD_CH_DIMM1, &dimm_dod[1]);
638                 pci_read_config_dword(pvt->pci_ch[i][1],
639                                 MC_DOD_CH_DIMM2, &dimm_dod[2]);
640
641                 debugf0("Ch%d phy rd%d, wr%d (0x%08x): "
642                         "%d ranks, %cDIMMs\n",
643                         i,
644                         RDLCH(pvt->info.ch_map, i), WRLCH(pvt->info.ch_map, i),
645                         data,
646                         pvt->channel[i].ranks,
647                         (data & REGISTERED_DIMM) ? 'R' : 'U');
648
649                 for (j = 0; j < 3; j++) {
650                         u32 banks, ranks, rows, cols;
651                         u32 size, npages;
652
653                         if (!DIMM_PRESENT(dimm_dod[j]))
654                                 continue;
655
656                         banks = numbank(MC_DOD_NUMBANK(dimm_dod[j]));
657                         ranks = numrank(MC_DOD_NUMRANK(dimm_dod[j]));
658                         rows = numrow(MC_DOD_NUMROW(dimm_dod[j]));
659                         cols = numcol(MC_DOD_NUMCOL(dimm_dod[j]));
660
661                         /* DDR3 has 8 I/O banks */
662                         size = (rows * cols * banks * ranks) >> (20 - 3);
663
664                         pvt->channel[i].dimms++;
665
666                         debugf0("\tdimm %d %d Mb offset: %x, "
667                                 "bank: %d, rank: %d, row: %#x, col: %#x\n",
668                                 j, size,
669                                 RANKOFFSET(dimm_dod[j]),
670                                 banks, ranks, rows, cols);
671
672                         npages = MiB_TO_PAGES(size);
673
674                         csr = &mci->csrows[*csrow];
675                         csr->first_page = last_page + 1;
676                         last_page += npages;
677                         csr->last_page = last_page;
678                         csr->nr_pages = npages;
679
680                         csr->page_mask = 0;
681                         csr->grain = 8;
682                         csr->csrow_idx = *csrow;
683                         csr->nr_channels = 1;
684
685                         csr->channels[0].chan_idx = i;
686                         csr->channels[0].ce_count = 0;
687
688                         pvt->csrow_map[i][j] = *csrow;
689
690                         switch (banks) {
691                         case 4:
692                                 csr->dtype = DEV_X4;
693                                 break;
694                         case 8:
695                                 csr->dtype = DEV_X8;
696                                 break;
697                         case 16:
698                                 csr->dtype = DEV_X16;
699                                 break;
700                         default:
701                                 csr->dtype = DEV_UNKNOWN;
702                         }
703
704                         csr->edac_mode = mode;
705                         csr->mtype = mtype;
706
707                         (*csrow)++;
708                 }
709
710                 pci_read_config_dword(pdev, MC_SAG_CH_0, &value[0]);
711                 pci_read_config_dword(pdev, MC_SAG_CH_1, &value[1]);
712                 pci_read_config_dword(pdev, MC_SAG_CH_2, &value[2]);
713                 pci_read_config_dword(pdev, MC_SAG_CH_3, &value[3]);
714                 pci_read_config_dword(pdev, MC_SAG_CH_4, &value[4]);
715                 pci_read_config_dword(pdev, MC_SAG_CH_5, &value[5]);
716                 pci_read_config_dword(pdev, MC_SAG_CH_6, &value[6]);
717                 pci_read_config_dword(pdev, MC_SAG_CH_7, &value[7]);
718                 debugf1("\t[%i] DIVBY3\tREMOVED\tOFFSET\n", i);
719                 for (j = 0; j < 8; j++)
720                         debugf1("\t\t%#x\t%#x\t%#x\n",
721                                 (value[j] >> 27) & 0x1,
722                                 (value[j] >> 24) & 0x7,
723                                 (value[j] && ((1 << 24) - 1)));
724         }
725
726         return 0;
727 }
728
729 /****************************************************************************
730                         Error insertion routines
731  ****************************************************************************/
732
733 /* The i7core has independent error injection features per channel.
734    However, to have a simpler code, we don't allow enabling error injection
735    on more than one channel.
736    Also, since a change at an inject parameter will be applied only at enable,
737    we're disabling error injection on all write calls to the sysfs nodes that
738    controls the error code injection.
739  */
740 static int disable_inject(const struct mem_ctl_info *mci)
741 {
742         struct i7core_pvt *pvt = mci->pvt_info;
743
744         pvt->inject.enable = 0;
745
746         if (!pvt->pci_ch[pvt->inject.channel][0])
747                 return -ENODEV;
748
749         pci_write_config_dword(pvt->pci_ch[pvt->inject.channel][0],
750                                 MC_CHANNEL_ERROR_INJECT, 0);
751
752         return 0;
753 }
754
755 /*
756  * i7core inject inject.section
757  *
758  *      accept and store error injection inject.section value
759  *      bit 0 - refers to the lower 32-byte half cacheline
760  *      bit 1 - refers to the upper 32-byte half cacheline
761  */
762 static ssize_t i7core_inject_section_store(struct mem_ctl_info *mci,
763                                            const char *data, size_t count)
764 {
765         struct i7core_pvt *pvt = mci->pvt_info;
766         unsigned long value;
767         int rc;
768
769         if (pvt->inject.enable)
770                 disable_inject(mci);
771
772         rc = strict_strtoul(data, 10, &value);
773         if ((rc < 0) || (value > 3))
774                 return -EIO;
775
776         pvt->inject.section = (u32) value;
777         return count;
778 }
779
780 static ssize_t i7core_inject_section_show(struct mem_ctl_info *mci,
781                                               char *data)
782 {
783         struct i7core_pvt *pvt = mci->pvt_info;
784         return sprintf(data, "0x%08x\n", pvt->inject.section);
785 }
786
787 /*
788  * i7core inject.type
789  *
790  *      accept and store error injection inject.section value
791  *      bit 0 - repeat enable - Enable error repetition
792  *      bit 1 - inject ECC error
793  *      bit 2 - inject parity error
794  */
795 static ssize_t i7core_inject_type_store(struct mem_ctl_info *mci,
796                                         const char *data, size_t count)
797 {
798         struct i7core_pvt *pvt = mci->pvt_info;
799         unsigned long value;
800         int rc;
801
802         if (pvt->inject.enable)
803                 disable_inject(mci);
804
805         rc = strict_strtoul(data, 10, &value);
806         if ((rc < 0) || (value > 7))
807                 return -EIO;
808
809         pvt->inject.type = (u32) value;
810         return count;
811 }
812
813 static ssize_t i7core_inject_type_show(struct mem_ctl_info *mci,
814                                               char *data)
815 {
816         struct i7core_pvt *pvt = mci->pvt_info;
817         return sprintf(data, "0x%08x\n", pvt->inject.type);
818 }
819
820 /*
821  * i7core_inject_inject.eccmask_store
822  *
823  * The type of error (UE/CE) will depend on the inject.eccmask value:
824  *   Any bits set to a 1 will flip the corresponding ECC bit
825  *   Correctable errors can be injected by flipping 1 bit or the bits within
826  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
827  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
828  *   uncorrectable error to be injected.
829  */
830 static ssize_t i7core_inject_eccmask_store(struct mem_ctl_info *mci,
831                                         const char *data, size_t count)
832 {
833         struct i7core_pvt *pvt = mci->pvt_info;
834         unsigned long value;
835         int rc;
836
837         if (pvt->inject.enable)
838                 disable_inject(mci);
839
840         rc = strict_strtoul(data, 10, &value);
841         if (rc < 0)
842                 return -EIO;
843
844         pvt->inject.eccmask = (u32) value;
845         return count;
846 }
847
848 static ssize_t i7core_inject_eccmask_show(struct mem_ctl_info *mci,
849                                               char *data)
850 {
851         struct i7core_pvt *pvt = mci->pvt_info;
852         return sprintf(data, "0x%08x\n", pvt->inject.eccmask);
853 }
854
855 /*
856  * i7core_addrmatch
857  *
858  * The type of error (UE/CE) will depend on the inject.eccmask value:
859  *   Any bits set to a 1 will flip the corresponding ECC bit
860  *   Correctable errors can be injected by flipping 1 bit or the bits within
861  *   a symbol pair (2 consecutive aligned 8-bit pairs - i.e. 7:0 and 15:8 or
862  *   23:16 and 31:24). Flipping bits in two symbol pairs will cause an
863  *   uncorrectable error to be injected.
864  */
865
866 #define DECLARE_ADDR_MATCH(param, limit)                        \
867 static ssize_t i7core_inject_store_##param(                     \
868                 struct mem_ctl_info *mci,                       \
869                 const char *data, size_t count)                 \
870 {                                                               \
871         struct i7core_pvt *pvt;                                 \
872         long value;                                             \
873         int rc;                                                 \
874                                                                 \
875         debugf1("%s()\n", __func__);                            \
876         pvt = mci->pvt_info;                                    \
877                                                                 \
878         if (pvt->inject.enable)                                 \
879                 disable_inject(mci);                            \
880                                                                 \
881         if (!strcasecmp(data, "any") || !strcasecmp(data, "any\n"))\
882                 value = -1;                                     \
883         else {                                                  \
884                 rc = strict_strtoul(data, 10, &value);          \
885                 if ((rc < 0) || (value >= limit))               \
886                         return -EIO;                            \
887         }                                                       \
888                                                                 \
889         pvt->inject.param = value;                              \
890                                                                 \
891         return count;                                           \
892 }                                                               \
893                                                                 \
894 static ssize_t i7core_inject_show_##param(                      \
895                 struct mem_ctl_info *mci,                       \
896                 char *data)                                     \
897 {                                                               \
898         struct i7core_pvt *pvt;                                 \
899                                                                 \
900         pvt = mci->pvt_info;                                    \
901         debugf1("%s() pvt=%p\n", __func__, pvt);                \
902         if (pvt->inject.param < 0)                              \
903                 return sprintf(data, "any\n");                  \
904         else                                                    \
905                 return sprintf(data, "%d\n", pvt->inject.param);\
906 }
907
908 #define ATTR_ADDR_MATCH(param)                                  \
909         {                                                       \
910                 .attr = {                                       \
911                         .name = #param,                         \
912                         .mode = (S_IRUGO | S_IWUSR)             \
913                 },                                              \
914                 .show  = i7core_inject_show_##param,            \
915                 .store = i7core_inject_store_##param,           \
916         }
917
918 DECLARE_ADDR_MATCH(channel, 3);
919 DECLARE_ADDR_MATCH(dimm, 3);
920 DECLARE_ADDR_MATCH(rank, 4);
921 DECLARE_ADDR_MATCH(bank, 32);
922 DECLARE_ADDR_MATCH(page, 0x10000);
923 DECLARE_ADDR_MATCH(col, 0x4000);
924
925 static int write_and_test(struct pci_dev *dev, const int where, const u32 val)
926 {
927         u32 read;
928         int count;
929
930         debugf0("setting pci %02x:%02x.%x reg=%02x value=%08x\n",
931                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
932                 where, val);
933
934         for (count = 0; count < 10; count++) {
935                 if (count)
936                         msleep(100);
937                 pci_write_config_dword(dev, where, val);
938                 pci_read_config_dword(dev, where, &read);
939
940                 if (read == val)
941                         return 0;
942         }
943
944         i7core_printk(KERN_ERR, "Error during set pci %02x:%02x.%x reg=%02x "
945                 "write=%08x. Read=%08x\n",
946                 dev->bus->number, PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn),
947                 where, val, read);
948
949         return -EINVAL;
950 }
951
952 /*
953  * This routine prepares the Memory Controller for error injection.
954  * The error will be injected when some process tries to write to the
955  * memory that matches the given criteria.
956  * The criteria can be set in terms of a mask where dimm, rank, bank, page
957  * and col can be specified.
958  * A -1 value for any of the mask items will make the MCU to ignore
959  * that matching criteria for error injection.
960  *
961  * It should be noticed that the error will only happen after a write operation
962  * on a memory that matches the condition. if REPEAT_EN is not enabled at
963  * inject mask, then it will produce just one error. Otherwise, it will repeat
964  * until the injectmask would be cleaned.
965  *
966  * FIXME: This routine assumes that MAXNUMDIMMS value of MC_MAX_DOD
967  *    is reliable enough to check if the MC is using the
968  *    three channels. However, this is not clear at the datasheet.
969  */
970 static ssize_t i7core_inject_enable_store(struct mem_ctl_info *mci,
971                                        const char *data, size_t count)
972 {
973         struct i7core_pvt *pvt = mci->pvt_info;
974         u32 injectmask;
975         u64 mask = 0;
976         int  rc;
977         long enable;
978
979         if (!pvt->pci_ch[pvt->inject.channel][0])
980                 return 0;
981
982         rc = strict_strtoul(data, 10, &enable);
983         if ((rc < 0))
984                 return 0;
985
986         if (enable) {
987                 pvt->inject.enable = 1;
988         } else {
989                 disable_inject(mci);
990                 return count;
991         }
992
993         /* Sets pvt->inject.dimm mask */
994         if (pvt->inject.dimm < 0)
995                 mask |= 1LL << 41;
996         else {
997                 if (pvt->channel[pvt->inject.channel].dimms > 2)
998                         mask |= (pvt->inject.dimm & 0x3LL) << 35;
999                 else
1000                         mask |= (pvt->inject.dimm & 0x1LL) << 36;
1001         }
1002
1003         /* Sets pvt->inject.rank mask */
1004         if (pvt->inject.rank < 0)
1005                 mask |= 1LL << 40;
1006         else {
1007                 if (pvt->channel[pvt->inject.channel].dimms > 2)
1008                         mask |= (pvt->inject.rank & 0x1LL) << 34;
1009                 else
1010                         mask |= (pvt->inject.rank & 0x3LL) << 34;
1011         }
1012
1013         /* Sets pvt->inject.bank mask */
1014         if (pvt->inject.bank < 0)
1015                 mask |= 1LL << 39;
1016         else
1017                 mask |= (pvt->inject.bank & 0x15LL) << 30;
1018
1019         /* Sets pvt->inject.page mask */
1020         if (pvt->inject.page < 0)
1021                 mask |= 1LL << 38;
1022         else
1023                 mask |= (pvt->inject.page & 0xffff) << 14;
1024
1025         /* Sets pvt->inject.column mask */
1026         if (pvt->inject.col < 0)
1027                 mask |= 1LL << 37;
1028         else
1029                 mask |= (pvt->inject.col & 0x3fff);
1030
1031         /*
1032          * bit    0: REPEAT_EN
1033          * bits 1-2: MASK_HALF_CACHELINE
1034          * bit    3: INJECT_ECC
1035          * bit    4: INJECT_ADDR_PARITY
1036          */
1037
1038         injectmask = (pvt->inject.type & 1) |
1039                      (pvt->inject.section & 0x3) << 1 |
1040                      (pvt->inject.type & 0x6) << (3 - 1);
1041
1042         /* Unlock writes to registers - this register is write only */
1043         pci_write_config_dword(pvt->pci_noncore,
1044                                MC_CFG_CONTROL, 0x2);
1045
1046         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1047                                MC_CHANNEL_ADDR_MATCH, mask);
1048         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1049                                MC_CHANNEL_ADDR_MATCH + 4, mask >> 32L);
1050
1051         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1052                                MC_CHANNEL_ERROR_MASK, pvt->inject.eccmask);
1053
1054         write_and_test(pvt->pci_ch[pvt->inject.channel][0],
1055                                MC_CHANNEL_ERROR_INJECT, injectmask);
1056
1057         /*
1058          * This is something undocumented, based on my tests
1059          * Without writing 8 to this register, errors aren't injected. Not sure
1060          * why.
1061          */
1062         pci_write_config_dword(pvt->pci_noncore,
1063                                MC_CFG_CONTROL, 8);
1064
1065         debugf0("Error inject addr match 0x%016llx, ecc 0x%08x,"
1066                 " inject 0x%08x\n",
1067                 mask, pvt->inject.eccmask, injectmask);
1068
1069
1070         return count;
1071 }
1072
1073 static ssize_t i7core_inject_enable_show(struct mem_ctl_info *mci,
1074                                         char *data)
1075 {
1076         struct i7core_pvt *pvt = mci->pvt_info;
1077         u32 injectmask;
1078
1079         if (!pvt->pci_ch[pvt->inject.channel][0])
1080                 return 0;
1081
1082         pci_read_config_dword(pvt->pci_ch[pvt->inject.channel][0],
1083                                MC_CHANNEL_ERROR_INJECT, &injectmask);
1084
1085         debugf0("Inject error read: 0x%018x\n", injectmask);
1086
1087         if (injectmask & 0x0c)
1088                 pvt->inject.enable = 1;
1089
1090         return sprintf(data, "%d\n", pvt->inject.enable);
1091 }
1092
1093 #define DECLARE_COUNTER(param)                                  \
1094 static ssize_t i7core_show_counter_##param(                     \
1095                 struct mem_ctl_info *mci,                       \
1096                 char *data)                                     \
1097 {                                                               \
1098         struct i7core_pvt *pvt = mci->pvt_info;                 \
1099                                                                 \
1100         debugf1("%s() \n", __func__);                           \
1101         if (!pvt->ce_count_available || (pvt->is_registered))   \
1102                 return sprintf(data, "data unavailable\n");     \
1103         return sprintf(data, "%lu\n",                           \
1104                         pvt->udimm_ce_count[param]);            \
1105 }
1106
1107 #define ATTR_COUNTER(param)                                     \
1108         {                                                       \
1109                 .attr = {                                       \
1110                         .name = __stringify(udimm##param),      \
1111                         .mode = (S_IRUGO | S_IWUSR)             \
1112                 },                                              \
1113                 .show  = i7core_show_counter_##param            \
1114         }
1115
1116 DECLARE_COUNTER(0);
1117 DECLARE_COUNTER(1);
1118 DECLARE_COUNTER(2);
1119
1120 /*
1121  * Sysfs struct
1122  */
1123
1124 static const struct mcidev_sysfs_attribute i7core_addrmatch_attrs[] = {
1125         ATTR_ADDR_MATCH(channel),
1126         ATTR_ADDR_MATCH(dimm),
1127         ATTR_ADDR_MATCH(rank),
1128         ATTR_ADDR_MATCH(bank),
1129         ATTR_ADDR_MATCH(page),
1130         ATTR_ADDR_MATCH(col),
1131         { } /* End of list */
1132 };
1133
1134 static const struct mcidev_sysfs_group i7core_inject_addrmatch = {
1135         .name  = "inject_addrmatch",
1136         .mcidev_attr = i7core_addrmatch_attrs,
1137 };
1138
1139 static const struct mcidev_sysfs_attribute i7core_udimm_counters_attrs[] = {
1140         ATTR_COUNTER(0),
1141         ATTR_COUNTER(1),
1142         ATTR_COUNTER(2),
1143         { .attr = { .name = NULL } }
1144 };
1145
1146 static const struct mcidev_sysfs_group i7core_udimm_counters = {
1147         .name  = "all_channel_counts",
1148         .mcidev_attr = i7core_udimm_counters_attrs,
1149 };
1150
1151 static const struct mcidev_sysfs_attribute i7core_sysfs_rdimm_attrs[] = {
1152         {
1153                 .attr = {
1154                         .name = "inject_section",
1155                         .mode = (S_IRUGO | S_IWUSR)
1156                 },
1157                 .show  = i7core_inject_section_show,
1158                 .store = i7core_inject_section_store,
1159         }, {
1160                 .attr = {
1161                         .name = "inject_type",
1162                         .mode = (S_IRUGO | S_IWUSR)
1163                 },
1164                 .show  = i7core_inject_type_show,
1165                 .store = i7core_inject_type_store,
1166         }, {
1167                 .attr = {
1168                         .name = "inject_eccmask",
1169                         .mode = (S_IRUGO | S_IWUSR)
1170                 },
1171                 .show  = i7core_inject_eccmask_show,
1172                 .store = i7core_inject_eccmask_store,
1173         }, {
1174                 .grp = &i7core_inject_addrmatch,
1175         }, {
1176                 .attr = {
1177                         .name = "inject_enable",
1178                         .mode = (S_IRUGO | S_IWUSR)
1179                 },
1180                 .show  = i7core_inject_enable_show,
1181                 .store = i7core_inject_enable_store,
1182         },
1183         { }     /* End of list */
1184 };
1185
1186 static const struct mcidev_sysfs_attribute i7core_sysfs_udimm_attrs[] = {
1187         {
1188                 .attr = {
1189                         .name = "inject_section",
1190                         .mode = (S_IRUGO | S_IWUSR)
1191                 },
1192                 .show  = i7core_inject_section_show,
1193                 .store = i7core_inject_section_store,
1194         }, {
1195                 .attr = {
1196                         .name = "inject_type",
1197                         .mode = (S_IRUGO | S_IWUSR)
1198                 },
1199                 .show  = i7core_inject_type_show,
1200                 .store = i7core_inject_type_store,
1201         }, {
1202                 .attr = {
1203                         .name = "inject_eccmask",
1204                         .mode = (S_IRUGO | S_IWUSR)
1205                 },
1206                 .show  = i7core_inject_eccmask_show,
1207                 .store = i7core_inject_eccmask_store,
1208         }, {
1209                 .grp = &i7core_inject_addrmatch,
1210         }, {
1211                 .attr = {
1212                         .name = "inject_enable",
1213                         .mode = (S_IRUGO | S_IWUSR)
1214                 },
1215                 .show  = i7core_inject_enable_show,
1216                 .store = i7core_inject_enable_store,
1217         }, {
1218                 .grp = &i7core_udimm_counters,
1219         },
1220         { }     /* End of list */
1221 };
1222
1223 /****************************************************************************
1224         Device initialization routines: put/get, init/exit
1225  ****************************************************************************/
1226
1227 /*
1228  *      i7core_put_devices      'put' all the devices that we have
1229  *                              reserved via 'get'
1230  */
1231 static void i7core_put_devices(struct i7core_dev *i7core_dev)
1232 {
1233         int i;
1234
1235         debugf0(__FILE__ ": %s()\n", __func__);
1236         for (i = 0; i < i7core_dev->n_devs; i++) {
1237                 struct pci_dev *pdev = i7core_dev->pdev[i];
1238                 if (!pdev)
1239                         continue;
1240                 debugf0("Removing dev %02x:%02x.%d\n",
1241                         pdev->bus->number,
1242                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
1243                 pci_dev_put(pdev);
1244         }
1245         kfree(i7core_dev->pdev);
1246 }
1247
1248 static void i7core_put_all_devices(void)
1249 {
1250         struct i7core_dev *i7core_dev, *tmp;
1251
1252         list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
1253                 i7core_put_devices(i7core_dev);
1254                 list_del(&i7core_dev->list);
1255                 kfree(i7core_dev);
1256         }
1257 }
1258
1259 static void __init i7core_xeon_pci_fixup(const struct pci_id_table *table)
1260 {
1261         struct pci_dev *pdev = NULL;
1262         int i;
1263
1264         /*
1265          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core pci buses
1266          * aren't announced by acpi. So, we need to use a legacy scan probing
1267          * to detect them
1268          */
1269         while (table && table->descr) {
1270                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, table->descr[0].dev_id, NULL);
1271                 if (unlikely(!pdev)) {
1272                         for (i = 0; i < MAX_SOCKET_BUSES; i++)
1273                                 pcibios_scan_specific_bus(255-i);
1274                 }
1275                 pci_dev_put(pdev);
1276                 table++;
1277         }
1278 }
1279
1280 static unsigned i7core_pci_lastbus(void)
1281 {
1282         int last_bus = 0, bus;
1283         struct pci_bus *b = NULL;
1284
1285         while ((b = pci_find_next_bus(b)) != NULL) {
1286                 bus = b->number;
1287                 debugf0("Found bus %d\n", bus);
1288                 if (bus > last_bus)
1289                         last_bus = bus;
1290         }
1291
1292         debugf0("Last bus %d\n", last_bus);
1293
1294         return last_bus;
1295 }
1296
1297 /*
1298  *      i7core_get_devices      Find and perform 'get' operation on the MCH's
1299  *                      device/functions we want to reference for this driver
1300  *
1301  *                      Need to 'get' device 16 func 1 and func 2
1302  */
1303 int i7core_get_onedevice(struct pci_dev **prev, const int devno,
1304                          const struct pci_id_descr *dev_descr,
1305                          const unsigned n_devs,
1306                          const unsigned last_bus)
1307 {
1308         struct i7core_dev *i7core_dev;
1309
1310         struct pci_dev *pdev = NULL;
1311         u8 bus = 0;
1312         u8 socket = 0;
1313
1314         pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1315                               dev_descr->dev_id, *prev);
1316
1317         /*
1318          * On Xeon 55xx, the Intel Quckpath Arch Generic Non-core regs
1319          * is at addr 8086:2c40, instead of 8086:2c41. So, we need
1320          * to probe for the alternate address in case of failure
1321          */
1322         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_I7_NONCORE && !pdev)
1323                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1324                                       PCI_DEVICE_ID_INTEL_I7_NONCORE_ALT, *prev);
1325
1326         if (dev_descr->dev_id == PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE && !pdev)
1327                 pdev = pci_get_device(PCI_VENDOR_ID_INTEL,
1328                                       PCI_DEVICE_ID_INTEL_LYNNFIELD_NONCORE_ALT,
1329                                       *prev);
1330
1331         if (!pdev) {
1332                 if (*prev) {
1333                         *prev = pdev;
1334                         return 0;
1335                 }
1336
1337                 if (dev_descr->optional)
1338                         return 0;
1339
1340                 if (devno == 0)
1341                         return -ENODEV;
1342
1343                 i7core_printk(KERN_INFO,
1344                         "Device not found: dev %02x.%d PCI ID %04x:%04x\n",
1345                         dev_descr->dev, dev_descr->func,
1346                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1347
1348                 /* End of list, leave */
1349                 return -ENODEV;
1350         }
1351         bus = pdev->bus->number;
1352
1353         socket = last_bus - bus;
1354
1355         i7core_dev = get_i7core_dev(socket);
1356         if (!i7core_dev) {
1357                 i7core_dev = kzalloc(sizeof(*i7core_dev), GFP_KERNEL);
1358                 if (!i7core_dev)
1359                         return -ENOMEM;
1360                 i7core_dev->pdev = kzalloc(sizeof(*i7core_dev->pdev) * n_devs,
1361                                            GFP_KERNEL);
1362                 if (!i7core_dev->pdev) {
1363                         kfree(i7core_dev);
1364                         return -ENOMEM;
1365                 }
1366                 i7core_dev->socket = socket;
1367                 i7core_dev->n_devs = n_devs;
1368                 list_add_tail(&i7core_dev->list, &i7core_edac_list);
1369         }
1370
1371         if (i7core_dev->pdev[devno]) {
1372                 i7core_printk(KERN_ERR,
1373                         "Duplicated device for "
1374                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1375                         bus, dev_descr->dev, dev_descr->func,
1376                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1377                 pci_dev_put(pdev);
1378                 return -ENODEV;
1379         }
1380
1381         i7core_dev->pdev[devno] = pdev;
1382
1383         /* Sanity check */
1384         if (unlikely(PCI_SLOT(pdev->devfn) != dev_descr->dev ||
1385                         PCI_FUNC(pdev->devfn) != dev_descr->func)) {
1386                 i7core_printk(KERN_ERR,
1387                         "Device PCI ID %04x:%04x "
1388                         "has dev %02x:%02x.%d instead of dev %02x:%02x.%d\n",
1389                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id,
1390                         bus, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1391                         bus, dev_descr->dev, dev_descr->func);
1392                 return -ENODEV;
1393         }
1394
1395         /* Be sure that the device is enabled */
1396         if (unlikely(pci_enable_device(pdev) < 0)) {
1397                 i7core_printk(KERN_ERR,
1398                         "Couldn't enable "
1399                         "dev %02x:%02x.%d PCI ID %04x:%04x\n",
1400                         bus, dev_descr->dev, dev_descr->func,
1401                         PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1402                 return -ENODEV;
1403         }
1404
1405         debugf0("Detected socket %d dev %02x:%02x.%d PCI ID %04x:%04x\n",
1406                 socket, bus, dev_descr->dev,
1407                 dev_descr->func,
1408                 PCI_VENDOR_ID_INTEL, dev_descr->dev_id);
1409
1410         *prev = pdev;
1411
1412         return 0;
1413 }
1414
1415 static int i7core_get_devices(const struct pci_id_table *table)
1416 {
1417         int i, rc, last_bus;
1418         struct pci_dev *pdev = NULL;
1419         const struct pci_id_descr *dev_descr;
1420
1421         last_bus = i7core_pci_lastbus();
1422
1423         while (table && table->descr) {
1424                 dev_descr = table->descr;
1425                 for (i = 0; i < table->n_devs; i++) {
1426                         pdev = NULL;
1427                         do {
1428                                 rc = i7core_get_onedevice(&pdev, i,
1429                                                           &dev_descr[i],
1430                                                           table->n_devs,
1431                                                           last_bus);
1432                                 if (rc < 0) {
1433                                         if (i == 0) {
1434                                                 i = table->n_devs;
1435                                                 break;
1436                                         }
1437                                         i7core_put_all_devices();
1438                                         return -ENODEV;
1439                                 }
1440                         } while (pdev);
1441                 }
1442                 table++;
1443         }
1444
1445         return 0;
1446 }
1447
1448 static int mci_bind_devs(struct mem_ctl_info *mci,
1449                          struct i7core_dev *i7core_dev)
1450 {
1451         struct i7core_pvt *pvt = mci->pvt_info;
1452         struct pci_dev *pdev;
1453         int i, func, slot;
1454
1455         /* Associates i7core_dev and mci for future usage */
1456         pvt->i7core_dev = i7core_dev;
1457         i7core_dev->mci = mci;
1458
1459         pvt->is_registered = 0;
1460         for (i = 0; i < i7core_dev->n_devs; i++) {
1461                 pdev = i7core_dev->pdev[i];
1462                 if (!pdev)
1463                         continue;
1464
1465                 func = PCI_FUNC(pdev->devfn);
1466                 slot = PCI_SLOT(pdev->devfn);
1467                 if (slot == 3) {
1468                         if (unlikely(func > MAX_MCR_FUNC))
1469                                 goto error;
1470                         pvt->pci_mcr[func] = pdev;
1471                 } else if (likely(slot >= 4 && slot < 4 + NUM_CHANS)) {
1472                         if (unlikely(func > MAX_CHAN_FUNC))
1473                                 goto error;
1474                         pvt->pci_ch[slot - 4][func] = pdev;
1475                 } else if (!slot && !func)
1476                         pvt->pci_noncore = pdev;
1477                 else
1478                         goto error;
1479
1480                 debugf0("Associated fn %d.%d, dev = %p, socket %d\n",
1481                         PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn),
1482                         pdev, i7core_dev->socket);
1483
1484                 if (PCI_SLOT(pdev->devfn) == 3 &&
1485                         PCI_FUNC(pdev->devfn) == 2)
1486                         pvt->is_registered = 1;
1487         }
1488
1489         return 0;
1490
1491 error:
1492         i7core_printk(KERN_ERR, "Device %d, function %d "
1493                       "is out of the expected range\n",
1494                       slot, func);
1495         return -EINVAL;
1496 }
1497
1498 /****************************************************************************
1499                         Error check routines
1500  ****************************************************************************/
1501 static void i7core_rdimm_update_csrow(struct mem_ctl_info *mci,
1502                                       const int chan,
1503                                       const int dimm,
1504                                       const int add)
1505 {
1506         char *msg;
1507         struct i7core_pvt *pvt = mci->pvt_info;
1508         int row = pvt->csrow_map[chan][dimm], i;
1509
1510         for (i = 0; i < add; i++) {
1511                 msg = kasprintf(GFP_KERNEL, "Corrected error "
1512                                 "(Socket=%d channel=%d dimm=%d)",
1513                                 pvt->i7core_dev->socket, chan, dimm);
1514
1515                 edac_mc_handle_fbd_ce(mci, row, 0, msg);
1516                 kfree (msg);
1517         }
1518 }
1519
1520 static void i7core_rdimm_update_ce_count(struct mem_ctl_info *mci,
1521                                          const int chan,
1522                                          const int new0,
1523                                          const int new1,
1524                                          const int new2)
1525 {
1526         struct i7core_pvt *pvt = mci->pvt_info;
1527         int add0 = 0, add1 = 0, add2 = 0;
1528         /* Updates CE counters if it is not the first time here */
1529         if (pvt->ce_count_available) {
1530                 /* Updates CE counters */
1531
1532                 add2 = new2 - pvt->rdimm_last_ce_count[chan][2];
1533                 add1 = new1 - pvt->rdimm_last_ce_count[chan][1];
1534                 add0 = new0 - pvt->rdimm_last_ce_count[chan][0];
1535
1536                 if (add2 < 0)
1537                         add2 += 0x7fff;
1538                 pvt->rdimm_ce_count[chan][2] += add2;
1539
1540                 if (add1 < 0)
1541                         add1 += 0x7fff;
1542                 pvt->rdimm_ce_count[chan][1] += add1;
1543
1544                 if (add0 < 0)
1545                         add0 += 0x7fff;
1546                 pvt->rdimm_ce_count[chan][0] += add0;
1547         } else
1548                 pvt->ce_count_available = 1;
1549
1550         /* Store the new values */
1551         pvt->rdimm_last_ce_count[chan][2] = new2;
1552         pvt->rdimm_last_ce_count[chan][1] = new1;
1553         pvt->rdimm_last_ce_count[chan][0] = new0;
1554
1555         /*updated the edac core */
1556         if (add0 != 0)
1557                 i7core_rdimm_update_csrow(mci, chan, 0, add0);
1558         if (add1 != 0)
1559                 i7core_rdimm_update_csrow(mci, chan, 1, add1);
1560         if (add2 != 0)
1561                 i7core_rdimm_update_csrow(mci, chan, 2, add2);
1562
1563 }
1564
1565 static void i7core_rdimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1566 {
1567         struct i7core_pvt *pvt = mci->pvt_info;
1568         u32 rcv[3][2];
1569         int i, new0, new1, new2;
1570
1571         /*Read DEV 3: FUN 2:  MC_COR_ECC_CNT regs directly*/
1572         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_0,
1573                                                                 &rcv[0][0]);
1574         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_1,
1575                                                                 &rcv[0][1]);
1576         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_2,
1577                                                                 &rcv[1][0]);
1578         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_3,
1579                                                                 &rcv[1][1]);
1580         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_4,
1581                                                                 &rcv[2][0]);
1582         pci_read_config_dword(pvt->pci_mcr[2], MC_COR_ECC_CNT_5,
1583                                                                 &rcv[2][1]);
1584         for (i = 0 ; i < 3; i++) {
1585                 debugf3("MC_COR_ECC_CNT%d = 0x%x; MC_COR_ECC_CNT%d = 0x%x\n",
1586                         (i * 2), rcv[i][0], (i * 2) + 1, rcv[i][1]);
1587                 /*if the channel has 3 dimms*/
1588                 if (pvt->channel[i].dimms > 2) {
1589                         new0 = DIMM_BOT_COR_ERR(rcv[i][0]);
1590                         new1 = DIMM_TOP_COR_ERR(rcv[i][0]);
1591                         new2 = DIMM_BOT_COR_ERR(rcv[i][1]);
1592                 } else {
1593                         new0 = DIMM_TOP_COR_ERR(rcv[i][0]) +
1594                                         DIMM_BOT_COR_ERR(rcv[i][0]);
1595                         new1 = DIMM_TOP_COR_ERR(rcv[i][1]) +
1596                                         DIMM_BOT_COR_ERR(rcv[i][1]);
1597                         new2 = 0;
1598                 }
1599
1600                 i7core_rdimm_update_ce_count(mci, i, new0, new1, new2);
1601         }
1602 }
1603
1604 /* This function is based on the device 3 function 4 registers as described on:
1605  * Intel Xeon Processor 5500 Series Datasheet Volume 2
1606  *      http://www.intel.com/Assets/PDF/datasheet/321322.pdf
1607  * also available at:
1608  *      http://www.arrownac.com/manufacturers/intel/s/nehalem/5500-datasheet-v2.pdf
1609  */
1610 static void i7core_udimm_check_mc_ecc_err(struct mem_ctl_info *mci)
1611 {
1612         struct i7core_pvt *pvt = mci->pvt_info;
1613         u32 rcv1, rcv0;
1614         int new0, new1, new2;
1615
1616         if (!pvt->pci_mcr[4]) {
1617                 debugf0("%s MCR registers not found\n", __func__);
1618                 return;
1619         }
1620
1621         /* Corrected test errors */
1622         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV1, &rcv1);
1623         pci_read_config_dword(pvt->pci_mcr[4], MC_TEST_ERR_RCV0, &rcv0);
1624
1625         /* Store the new values */
1626         new2 = DIMM2_COR_ERR(rcv1);
1627         new1 = DIMM1_COR_ERR(rcv0);
1628         new0 = DIMM0_COR_ERR(rcv0);
1629
1630         /* Updates CE counters if it is not the first time here */
1631         if (pvt->ce_count_available) {
1632                 /* Updates CE counters */
1633                 int add0, add1, add2;
1634
1635                 add2 = new2 - pvt->udimm_last_ce_count[2];
1636                 add1 = new1 - pvt->udimm_last_ce_count[1];
1637                 add0 = new0 - pvt->udimm_last_ce_count[0];
1638
1639                 if (add2 < 0)
1640                         add2 += 0x7fff;
1641                 pvt->udimm_ce_count[2] += add2;
1642
1643                 if (add1 < 0)
1644                         add1 += 0x7fff;
1645                 pvt->udimm_ce_count[1] += add1;
1646
1647                 if (add0 < 0)
1648                         add0 += 0x7fff;
1649                 pvt->udimm_ce_count[0] += add0;
1650
1651                 if (add0 | add1 | add2)
1652                         i7core_printk(KERN_ERR, "New Corrected error(s): "
1653                                       "dimm0: +%d, dimm1: +%d, dimm2 +%d\n",
1654                                       add0, add1, add2);
1655         } else
1656                 pvt->ce_count_available = 1;
1657
1658         /* Store the new values */
1659         pvt->udimm_last_ce_count[2] = new2;
1660         pvt->udimm_last_ce_count[1] = new1;
1661         pvt->udimm_last_ce_count[0] = new0;
1662 }
1663
1664 /*
1665  * According with tables E-11 and E-12 of chapter E.3.3 of Intel 64 and IA-32
1666  * Architectures Software Developer’s Manual Volume 3B.
1667  * Nehalem are defined as family 0x06, model 0x1a
1668  *
1669  * The MCA registers used here are the following ones:
1670  *     struct mce field MCA Register
1671  *     m->status        MSR_IA32_MC8_STATUS
1672  *     m->addr          MSR_IA32_MC8_ADDR
1673  *     m->misc          MSR_IA32_MC8_MISC
1674  * In the case of Nehalem, the error information is masked at .status and .misc
1675  * fields
1676  */
1677 static void i7core_mce_output_error(struct mem_ctl_info *mci,
1678                                     const struct mce *m)
1679 {
1680         struct i7core_pvt *pvt = mci->pvt_info;
1681         char *type, *optype, *err, *msg;
1682         unsigned long error = m->status & 0x1ff0000l;
1683         u32 optypenum = (m->status >> 4) & 0x07;
1684         u32 core_err_cnt = (m->status >> 38) && 0x7fff;
1685         u32 dimm = (m->misc >> 16) & 0x3;
1686         u32 channel = (m->misc >> 18) & 0x3;
1687         u32 syndrome = m->misc >> 32;
1688         u32 errnum = find_first_bit(&error, 32);
1689         int csrow;
1690
1691         if (m->mcgstatus & 1)
1692                 type = "FATAL";
1693         else
1694                 type = "NON_FATAL";
1695
1696         switch (optypenum) {
1697         case 0:
1698                 optype = "generic undef request";
1699                 break;
1700         case 1:
1701                 optype = "read error";
1702                 break;
1703         case 2:
1704                 optype = "write error";
1705                 break;
1706         case 3:
1707                 optype = "addr/cmd error";
1708                 break;
1709         case 4:
1710                 optype = "scrubbing error";
1711                 break;
1712         default:
1713                 optype = "reserved";
1714                 break;
1715         }
1716
1717         switch (errnum) {
1718         case 16:
1719                 err = "read ECC error";
1720                 break;
1721         case 17:
1722                 err = "RAS ECC error";
1723                 break;
1724         case 18:
1725                 err = "write parity error";
1726                 break;
1727         case 19:
1728                 err = "redundacy loss";
1729                 break;
1730         case 20:
1731                 err = "reserved";
1732                 break;
1733         case 21:
1734                 err = "memory range error";
1735                 break;
1736         case 22:
1737                 err = "RTID out of range";
1738                 break;
1739         case 23:
1740                 err = "address parity error";
1741                 break;
1742         case 24:
1743                 err = "byte enable parity error";
1744                 break;
1745         default:
1746                 err = "unknown";
1747         }
1748
1749         /* FIXME: should convert addr into bank and rank information */
1750         msg = kasprintf(GFP_ATOMIC,
1751                 "%s (addr = 0x%08llx, cpu=%d, Dimm=%d, Channel=%d, "
1752                 "syndrome=0x%08x, count=%d, Err=%08llx:%08llx (%s: %s))\n",
1753                 type, (long long) m->addr, m->cpu, dimm, channel,
1754                 syndrome, core_err_cnt, (long long)m->status,
1755                 (long long)m->misc, optype, err);
1756
1757         debugf0("%s", msg);
1758
1759         csrow = pvt->csrow_map[channel][dimm];
1760
1761         /* Call the helper to output message */
1762         if (m->mcgstatus & 1)
1763                 edac_mc_handle_fbd_ue(mci, csrow, 0,
1764                                 0 /* FIXME: should be channel here */, msg);
1765         else if (!pvt->is_registered)
1766                 edac_mc_handle_fbd_ce(mci, csrow,
1767                                 0 /* FIXME: should be channel here */, msg);
1768
1769         kfree(msg);
1770 }
1771
1772 /*
1773  *      i7core_check_error      Retrieve and process errors reported by the
1774  *                              hardware. Called by the Core module.
1775  */
1776 static void i7core_check_error(struct mem_ctl_info *mci)
1777 {
1778         struct i7core_pvt *pvt = mci->pvt_info;
1779         int i;
1780         unsigned count = 0;
1781         struct mce *m;
1782
1783         /*
1784          * MCE first step: Copy all mce errors into a temporary buffer
1785          * We use a double buffering here, to reduce the risk of
1786          * loosing an error.
1787          */
1788         smp_rmb();
1789         count = (pvt->mce_out + MCE_LOG_LEN - pvt->mce_in)
1790                 % MCE_LOG_LEN;
1791         if (!count)
1792                 goto check_ce_error;
1793
1794         m = pvt->mce_outentry;
1795         if (pvt->mce_in + count > MCE_LOG_LEN) {
1796                 unsigned l = MCE_LOG_LEN - pvt->mce_in;
1797
1798                 memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * l);
1799                 smp_wmb();
1800                 pvt->mce_in = 0;
1801                 count -= l;
1802                 m += l;
1803         }
1804         memcpy(m, &pvt->mce_entry[pvt->mce_in], sizeof(*m) * count);
1805         smp_wmb();
1806         pvt->mce_in += count;
1807
1808         smp_rmb();
1809         if (pvt->mce_overrun) {
1810                 i7core_printk(KERN_ERR, "Lost %d memory errors\n",
1811                               pvt->mce_overrun);
1812                 smp_wmb();
1813                 pvt->mce_overrun = 0;
1814         }
1815
1816         /*
1817          * MCE second step: parse errors and display
1818          */
1819         for (i = 0; i < count; i++)
1820                 i7core_mce_output_error(mci, &pvt->mce_outentry[i]);
1821
1822         /*
1823          * Now, let's increment CE error counts
1824          */
1825 check_ce_error:
1826         if (!pvt->is_registered)
1827                 i7core_udimm_check_mc_ecc_err(mci);
1828         else
1829                 i7core_rdimm_check_mc_ecc_err(mci);
1830 }
1831
1832 /*
1833  * i7core_mce_check_error       Replicates mcelog routine to get errors
1834  *                              This routine simply queues mcelog errors, and
1835  *                              return. The error itself should be handled later
1836  *                              by i7core_check_error.
1837  * WARNING: As this routine should be called at NMI time, extra care should
1838  * be taken to avoid deadlocks, and to be as fast as possible.
1839  */
1840 static int i7core_mce_check_error(void *priv, struct mce *mce)
1841 {
1842         struct mem_ctl_info *mci = priv;
1843         struct i7core_pvt *pvt = mci->pvt_info;
1844
1845         /*
1846          * Just let mcelog handle it if the error is
1847          * outside the memory controller
1848          */
1849         if (((mce->status & 0xffff) >> 7) != 1)
1850                 return 0;
1851
1852         /* Bank 8 registers are the only ones that we know how to handle */
1853         if (mce->bank != 8)
1854                 return 0;
1855
1856 #ifdef CONFIG_SMP
1857         /* Only handle if it is the right mc controller */
1858         if (cpu_data(mce->cpu).phys_proc_id != pvt->i7core_dev->socket)
1859                 return 0;
1860 #endif
1861
1862         smp_rmb();
1863         if ((pvt->mce_out + 1) % MCE_LOG_LEN == pvt->mce_in) {
1864                 smp_wmb();
1865                 pvt->mce_overrun++;
1866                 return 0;
1867         }
1868
1869         /* Copy memory error at the ringbuffer */
1870         memcpy(&pvt->mce_entry[pvt->mce_out], mce, sizeof(*mce));
1871         smp_wmb();
1872         pvt->mce_out = (pvt->mce_out + 1) % MCE_LOG_LEN;
1873
1874         /* Handle fatal errors immediately */
1875         if (mce->mcgstatus & 1)
1876                 i7core_check_error(mci);
1877
1878         /* Advice mcelog that the error were handled */
1879         return 1;
1880 }
1881
1882 static int i7core_register_mci(struct i7core_dev *i7core_dev,
1883                                const int num_channels, const int num_csrows)
1884 {
1885         struct mem_ctl_info *mci;
1886         struct i7core_pvt *pvt;
1887         int csrow = 0;
1888         int rc;
1889
1890         /* allocate a new MC control structure */
1891         mci = edac_mc_alloc(sizeof(*pvt), num_csrows, num_channels,
1892                             i7core_dev->socket);
1893         if (unlikely(!mci))
1894                 return -ENOMEM;
1895
1896         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
1897                 __func__, mci, &i7core_dev->pdev[0]->dev);
1898
1899         /* record ptr to the generic device */
1900         mci->dev = &i7core_dev->pdev[0]->dev;
1901
1902         pvt = mci->pvt_info;
1903         memset(pvt, 0, sizeof(*pvt));
1904
1905         /*
1906          * FIXME: how to handle RDDR3 at MCI level? It is possible to have
1907          * Mixed RDDR3/UDDR3 with Nehalem, provided that they are on different
1908          * memory channels
1909          */
1910         mci->mtype_cap = MEM_FLAG_DDR3;
1911         mci->edac_ctl_cap = EDAC_FLAG_NONE;
1912         mci->edac_cap = EDAC_FLAG_NONE;
1913         mci->mod_name = "i7core_edac.c";
1914         mci->mod_ver = I7CORE_REVISION;
1915         mci->ctl_name = kasprintf(GFP_KERNEL, "i7 core #%d",
1916                                   i7core_dev->socket);
1917         mci->dev_name = pci_name(i7core_dev->pdev[0]);
1918         mci->ctl_page_to_phys = NULL;
1919
1920         if (pvt->is_registered)
1921                 mci->mc_driver_sysfs_attributes = i7core_sysfs_rdimm_attrs;
1922         else
1923                 mci->mc_driver_sysfs_attributes = i7core_sysfs_udimm_attrs;
1924
1925         /* Set the function pointer to an actual operation function */
1926         mci->edac_check = i7core_check_error;
1927
1928         /* Store pci devices at mci for faster access */
1929         rc = mci_bind_devs(mci, i7core_dev);
1930         if (unlikely(rc < 0))
1931                 goto fail;
1932
1933         /* Get dimm basic config */
1934         get_dimm_config(mci, &csrow);
1935
1936         /* add this new MC control structure to EDAC's list of MCs */
1937         if (unlikely(edac_mc_add_mc(mci))) {
1938                 debugf0("MC: " __FILE__
1939                         ": %s(): failed edac_mc_add_mc()\n", __func__);
1940                 /* FIXME: perhaps some code should go here that disables error
1941                  * reporting if we just enabled it
1942                  */
1943
1944                 rc = -EINVAL;
1945                 goto fail;
1946         }
1947
1948         /* Default error mask is any memory */
1949         pvt->inject.channel = 0;
1950         pvt->inject.dimm = -1;
1951         pvt->inject.rank = -1;
1952         pvt->inject.bank = -1;
1953         pvt->inject.page = -1;
1954         pvt->inject.col = -1;
1955
1956         /* Registers on edac_mce in order to receive memory errors */
1957         pvt->edac_mce.priv = mci;
1958         pvt->edac_mce.check_error = i7core_mce_check_error;
1959
1960         /* allocating generic PCI control info */
1961         pvt->i7core_pci = edac_pci_create_generic_ctl(&i7core_dev->pdev[0]->dev,
1962                                                  EDAC_MOD_STR);
1963         if (unlikely(!pvt->i7core_pci)) {
1964                 printk(KERN_WARNING
1965                         "%s(): Unable to create PCI control\n",
1966                         __func__);
1967                 printk(KERN_WARNING
1968                         "%s(): PCI error report via EDAC not setup\n",
1969                         __func__);
1970         }
1971
1972         rc = edac_mce_register(&pvt->edac_mce);
1973         if (unlikely(rc < 0)) {
1974                 debugf0("MC: " __FILE__
1975                         ": %s(): failed edac_mce_register()\n", __func__);
1976         }
1977
1978 fail:
1979         if (rc < 0)
1980                 edac_mc_free(mci);
1981         return rc;
1982 }
1983
1984 /*
1985  *      i7core_probe    Probe for ONE instance of device to see if it is
1986  *                      present.
1987  *      return:
1988  *              0 for FOUND a device
1989  *              < 0 for error code
1990  */
1991
1992 static int __devinit i7core_probe(struct pci_dev *pdev,
1993                                   const struct pci_device_id *id)
1994 {
1995         int rc;
1996         struct i7core_dev *i7core_dev;
1997
1998         /* get the pci devices we want to reserve for our use */
1999         mutex_lock(&i7core_edac_lock);
2000
2001         /*
2002          * All memory controllers are allocated at the first pass.
2003          */
2004         if (unlikely(probed >= 1)) {
2005                 mutex_unlock(&i7core_edac_lock);
2006                 return -EINVAL;
2007         }
2008         probed++;
2009
2010         rc = i7core_get_devices(pci_dev_table);
2011         if (unlikely(rc < 0))
2012                 goto fail0;
2013
2014         list_for_each_entry(i7core_dev, &i7core_edac_list, list) {
2015                 int channels;
2016                 int csrows;
2017
2018                 /* Check the number of active and not disabled channels */
2019                 rc = i7core_get_active_channels(i7core_dev->socket,
2020                                                 &channels, &csrows);
2021                 if (unlikely(rc < 0))
2022                         goto fail1;
2023
2024                 rc = i7core_register_mci(i7core_dev, channels, csrows);
2025                 if (unlikely(rc < 0))
2026                         goto fail1;
2027         }
2028
2029         i7core_printk(KERN_INFO, "Driver loaded.\n");
2030
2031         mutex_unlock(&i7core_edac_lock);
2032         return 0;
2033
2034 fail1:
2035         i7core_put_all_devices();
2036 fail0:
2037         mutex_unlock(&i7core_edac_lock);
2038         return rc;
2039 }
2040
2041 /*
2042  *      i7core_remove   destructor for one instance of device
2043  *
2044  */
2045 static void __devexit i7core_remove(struct pci_dev *pdev)
2046 {
2047         struct mem_ctl_info *mci;
2048         struct i7core_dev *i7core_dev, *tmp;
2049         struct i7core_pvt *pvt;
2050
2051         debugf0(__FILE__ ": %s()\n", __func__);
2052
2053         /*
2054          * we have a trouble here: pdev value for removal will be wrong, since
2055          * it will point to the X58 register used to detect that the machine
2056          * is a Nehalem or upper design. However, due to the way several PCI
2057          * devices are grouped together to provide MC functionality, we need
2058          * to use a different method for releasing the devices
2059          */
2060
2061         mutex_lock(&i7core_edac_lock);
2062         list_for_each_entry_safe(i7core_dev, tmp, &i7core_edac_list, list) {
2063                 mci = find_mci_by_dev(&i7core_dev->pdev[0]->dev);
2064                 if (unlikely(!mci || !mci->pvt_info)) {
2065                         debugf0("MC: " __FILE__ ": %s(): dev = %p\n",
2066                                 __func__, &i7core_dev->pdev[0]->dev);
2067
2068                                 i7core_printk(KERN_ERR,
2069                                       "Couldn't find mci hanler\n");
2070                 } else {
2071                         pvt = mci->pvt_info;
2072                         i7core_dev = pvt->i7core_dev;
2073
2074                         debugf0("MC: " __FILE__ ": %s(): mci = %p, dev = %p\n",
2075                                 __func__, mci, &i7core_dev->pdev[0]->dev);
2076
2077                         /* Disable MCE NMI handler */
2078                         edac_mce_unregister(&pvt->edac_mce);
2079
2080                         /* Disable EDAC polling */
2081                         if (likely(pvt->i7core_pci))
2082                                 edac_pci_release_generic_ctl(pvt->i7core_pci);
2083                         else
2084                                 i7core_printk(KERN_ERR,
2085                                               "Couldn't find mem_ctl_info for socket %d\n",
2086                                               i7core_dev->socket);
2087                         pvt->i7core_pci = NULL;
2088
2089                         /* Remove MC sysfs nodes */
2090                         edac_mc_del_mc(&i7core_dev->pdev[0]->dev);
2091
2092                         debugf1("%s: free mci struct\n", mci->ctl_name);
2093                         kfree(mci->ctl_name);
2094                         edac_mc_free(mci);
2095
2096                         /* Release PCI resources */
2097                         i7core_put_devices(i7core_dev);
2098                 }
2099                 list_del(&i7core_dev->list);
2100                 kfree(i7core_dev);
2101         }
2102         probed--;
2103
2104         mutex_unlock(&i7core_edac_lock);
2105 }
2106
2107 MODULE_DEVICE_TABLE(pci, i7core_pci_tbl);
2108
2109 /*
2110  *      i7core_driver   pci_driver structure for this module
2111  *
2112  */
2113 static struct pci_driver i7core_driver = {
2114         .name     = "i7core_edac",
2115         .probe    = i7core_probe,
2116         .remove   = __devexit_p(i7core_remove),
2117         .id_table = i7core_pci_tbl,
2118 };
2119
2120 /*
2121  *      i7core_init             Module entry function
2122  *                      Try to initialize this module for its devices
2123  */
2124 static int __init i7core_init(void)
2125 {
2126         int pci_rc;
2127
2128         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2129
2130         /* Ensure that the OPSTATE is set correctly for POLL or NMI */
2131         opstate_init();
2132
2133         if (use_pci_fixup)
2134                 i7core_xeon_pci_fixup(pci_dev_table);
2135
2136         pci_rc = pci_register_driver(&i7core_driver);
2137
2138         if (pci_rc >= 0)
2139                 return 0;
2140
2141         i7core_printk(KERN_ERR, "Failed to register device with error %d.\n",
2142                       pci_rc);
2143
2144         return pci_rc;
2145 }
2146
2147 /*
2148  *      i7core_exit()   Module exit function
2149  *                      Unregister the driver
2150  */
2151 static void __exit i7core_exit(void)
2152 {
2153         debugf2("MC: " __FILE__ ": %s()\n", __func__);
2154         pci_unregister_driver(&i7core_driver);
2155 }
2156
2157 module_init(i7core_init);
2158 module_exit(i7core_exit);
2159
2160 MODULE_LICENSE("GPL");
2161 MODULE_AUTHOR("Mauro Carvalho Chehab <mchehab@redhat.com>");
2162 MODULE_AUTHOR("Red Hat Inc. (http://www.redhat.com)");
2163 MODULE_DESCRIPTION("MC Driver for Intel i7 Core memory controllers - "
2164                    I7CORE_REVISION);
2165
2166 module_param(edac_op_state, int, 0444);
2167 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");