1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
4 static struct edac_pci_ctl_info *amd64_ctl_pci;
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
16 static struct msr __percpu *msrs;
19 * count successfully initialized driver instances for setup_pci_device()
21 static atomic_t drv_instances = ATOMIC_INIT(0);
23 /* Per-node driver instances */
24 static struct mem_ctl_info **mcis;
25 static struct ecc_settings **ecc_stngs;
28 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
32 *FIXME: Produce a better mapping/linearisation.
35 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
38 { 0x01, 1600000000UL},
60 { 0x00, 0UL}, /* scrubbing off */
63 static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
68 err = pci_read_config_dword(pdev, offset, val);
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
76 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
81 err = pci_write_config_dword(pdev, offset, val);
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
91 * Depending on the family, F2 DCT reads need special handling:
93 * K8: has a single DCT only
95 * F10h: each DCT has its own set of regs
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
102 static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
108 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
111 static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
114 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
118 * Select DCT to which PCI cfg accesses are routed
120 static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
124 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
127 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
130 static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
135 if (addr >= 0x140 && addr <= 0x1a0) {
140 f15h_select_dct(pvt, dct);
142 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
146 * Memory scrubber control interface. For K8, memory scrubbing is handled by
147 * hardware and can involve L2 cache, dcache as well as the main memory. With
148 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
151 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
152 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
153 * bytes/sec for the setting.
155 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
156 * other archs, we might not have access to the caches directly.
160 * scan the scrub rate mapping table for a close or matching bandwidth value to
161 * issue. If requested is too big, then use last maximum value found.
163 static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
169 * map the configured rate (new_bw) to a value specific to the AMD64
170 * memory controller and apply to register. Search for the first
171 * bandwidth entry that is greater or equal than the setting requested
172 * and program that. If at last entry, turn off DRAM scrubbing.
174 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
175 * by falling back to the last element in scrubrates[].
177 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
179 * skip scrub rates which aren't recommended
180 * (see F10 BKDG, F3x58)
182 if (scrubrates[i].scrubval < min_rate)
185 if (scrubrates[i].bandwidth <= new_bw)
189 scrubval = scrubrates[i].scrubval;
191 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
194 return scrubrates[i].bandwidth;
199 static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
201 struct amd64_pvt *pvt = mci->pvt_info;
202 u32 min_scrubrate = 0x5;
204 if (boot_cpu_data.x86 == 0xf)
207 /* F15h Erratum #505 */
208 if (boot_cpu_data.x86 == 0x15)
209 f15h_select_dct(pvt, 0);
211 return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
214 static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
216 struct amd64_pvt *pvt = mci->pvt_info;
218 int i, retval = -EINVAL;
220 /* F15h Erratum #505 */
221 if (boot_cpu_data.x86 == 0x15)
222 f15h_select_dct(pvt, 0);
224 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
226 scrubval = scrubval & 0x001F;
228 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
229 if (scrubrates[i].scrubval == scrubval) {
230 retval = scrubrates[i].bandwidth;
238 * returns true if the SysAddr given by sys_addr matches the
239 * DRAM base/limit associated with node_id
241 static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
246 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
247 * all ones if the most significant implemented address bit is 1.
248 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
249 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
250 * Application Programming.
252 addr = sys_addr & 0x000000ffffffffffull;
254 return ((addr >= get_dram_base(pvt, nid)) &&
255 (addr <= get_dram_limit(pvt, nid)));
259 * Attempt to map a SysAddr to a node. On success, return a pointer to the
260 * mem_ctl_info structure for the node that the SysAddr maps to.
262 * On failure, return NULL.
264 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
267 struct amd64_pvt *pvt;
272 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
273 * 3.4.4.2) registers to map the SysAddr to a node ID.
278 * The value of this field should be the same for all DRAM Base
279 * registers. Therefore we arbitrarily choose to read it from the
280 * register for node 0.
282 intlv_en = dram_intlv_en(pvt, 0);
285 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
286 if (amd64_base_limit_match(pvt, sys_addr, node_id))
292 if (unlikely((intlv_en != 0x01) &&
293 (intlv_en != 0x03) &&
294 (intlv_en != 0x07))) {
295 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
299 bits = (((u32) sys_addr) >> 12) & intlv_en;
301 for (node_id = 0; ; ) {
302 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
303 break; /* intlv_sel field matches */
305 if (++node_id >= DRAM_RANGES)
309 /* sanity test for sys_addr */
310 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
311 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
312 "range for node %d with node interleaving enabled.\n",
313 __func__, sys_addr, node_id);
318 return edac_mc_find((int)node_id);
321 debugf2("sys_addr 0x%lx doesn't match any node\n",
322 (unsigned long)sys_addr);
328 * compute the CS base address of the @csrow on the DRAM controller @dct.
329 * For details see F2x[5C:40] in the processor's BKDG
331 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
332 u64 *base, u64 *mask)
334 u64 csbase, csmask, base_bits, mask_bits;
337 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
338 csbase = pvt->csels[dct].csbases[csrow];
339 csmask = pvt->csels[dct].csmasks[csrow];
340 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
341 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
344 csbase = pvt->csels[dct].csbases[csrow];
345 csmask = pvt->csels[dct].csmasks[csrow >> 1];
348 if (boot_cpu_data.x86 == 0x15)
349 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
351 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
354 *base = (csbase & base_bits) << addr_shift;
357 /* poke holes for the csmask */
358 *mask &= ~(mask_bits << addr_shift);
360 *mask |= (csmask & mask_bits) << addr_shift;
363 #define for_each_chip_select(i, dct, pvt) \
364 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
366 #define chip_select_base(i, dct, pvt) \
367 pvt->csels[dct].csbases[i]
369 #define for_each_chip_select_mask(i, dct, pvt) \
370 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
373 * @input_addr is an InputAddr associated with the node given by mci. Return the
374 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
376 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
378 struct amd64_pvt *pvt;
384 for_each_chip_select(csrow, 0, pvt) {
385 if (!csrow_enabled(csrow, 0, pvt))
388 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
392 if ((input_addr & mask) == (base & mask)) {
393 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
394 (unsigned long)input_addr, csrow,
400 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
401 (unsigned long)input_addr, pvt->mc_node_id);
407 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
408 * for the node represented by mci. Info is passed back in *hole_base,
409 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
410 * info is invalid. Info may be invalid for either of the following reasons:
412 * - The revision of the node is not E or greater. In this case, the DRAM Hole
413 * Address Register does not exist.
415 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
416 * indicating that its contents are not valid.
418 * The values passed back in *hole_base, *hole_offset, and *hole_size are
419 * complete 32-bit values despite the fact that the bitfields in the DHAR
420 * only represent bits 31-24 of the base and offset values.
422 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
423 u64 *hole_offset, u64 *hole_size)
425 struct amd64_pvt *pvt = mci->pvt_info;
428 /* only revE and later have the DRAM Hole Address Register */
429 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
430 debugf1(" revision %d for node %d does not support DHAR\n",
431 pvt->ext_model, pvt->mc_node_id);
435 /* valid for Fam10h and above */
436 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
437 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
441 if (!dhar_valid(pvt)) {
442 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
447 /* This node has Memory Hoisting */
449 /* +------------------+--------------------+--------------------+-----
450 * | memory | DRAM hole | relocated |
451 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
453 * | | | [0x100000000, |
454 * | | | (0x100000000+ |
455 * | | | (0xffffffff-x))] |
456 * +------------------+--------------------+--------------------+-----
458 * Above is a diagram of physical memory showing the DRAM hole and the
459 * relocated addresses from the DRAM hole. As shown, the DRAM hole
460 * starts at address x (the base address) and extends through address
461 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
462 * addresses in the hole so that they start at 0x100000000.
465 base = dhar_base(pvt);
468 *hole_size = (0x1ull << 32) - base;
470 if (boot_cpu_data.x86 > 0xf)
471 *hole_offset = f10_dhar_offset(pvt);
473 *hole_offset = k8_dhar_offset(pvt);
475 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
476 pvt->mc_node_id, (unsigned long)*hole_base,
477 (unsigned long)*hole_offset, (unsigned long)*hole_size);
481 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
484 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
485 * assumed that sys_addr maps to the node given by mci.
487 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
488 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
489 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
490 * then it is also involved in translating a SysAddr to a DramAddr. Sections
491 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
492 * These parts of the documentation are unclear. I interpret them as follows:
494 * When node n receives a SysAddr, it processes the SysAddr as follows:
496 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
497 * Limit registers for node n. If the SysAddr is not within the range
498 * specified by the base and limit values, then node n ignores the Sysaddr
499 * (since it does not map to node n). Otherwise continue to step 2 below.
501 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
502 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
503 * the range of relocated addresses (starting at 0x100000000) from the DRAM
504 * hole. If not, skip to step 3 below. Else get the value of the
505 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
506 * offset defined by this value from the SysAddr.
508 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
509 * Base register for node n. To obtain the DramAddr, subtract the base
510 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
512 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
514 struct amd64_pvt *pvt = mci->pvt_info;
515 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
518 dram_base = get_dram_base(pvt, pvt->mc_node_id);
520 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
523 if ((sys_addr >= (1ull << 32)) &&
524 (sys_addr < ((1ull << 32) + hole_size))) {
525 /* use DHAR to translate SysAddr to DramAddr */
526 dram_addr = sys_addr - hole_offset;
528 debugf2("using DHAR to translate SysAddr 0x%lx to "
530 (unsigned long)sys_addr,
531 (unsigned long)dram_addr);
538 * Translate the SysAddr to a DramAddr as shown near the start of
539 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
540 * only deals with 40-bit values. Therefore we discard bits 63-40 of
541 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
542 * discard are all 1s. Otherwise the bits we discard are all 0s. See
543 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
544 * Programmer's Manual Volume 1 Application Programming.
546 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
548 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
549 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
550 (unsigned long)dram_addr);
555 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
556 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
557 * for node interleaving.
559 static int num_node_interleave_bits(unsigned intlv_en)
561 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
564 BUG_ON(intlv_en > 7);
565 n = intlv_shift_table[intlv_en];
569 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
570 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
572 struct amd64_pvt *pvt;
579 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
580 * concerning translating a DramAddr to an InputAddr.
582 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
583 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
586 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
587 intlv_shift, (unsigned long)dram_addr,
588 (unsigned long)input_addr);
594 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
595 * assumed that @sys_addr maps to the node given by mci.
597 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
602 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
604 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
605 (unsigned long)sys_addr, (unsigned long)input_addr);
612 * @input_addr is an InputAddr associated with the node represented by mci.
613 * Translate @input_addr to a DramAddr and return the result.
615 static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
617 struct amd64_pvt *pvt;
618 unsigned node_id, intlv_shift;
623 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
624 * shows how to translate a DramAddr to an InputAddr. Here we reverse
625 * this procedure. When translating from a DramAddr to an InputAddr, the
626 * bits used for node interleaving are discarded. Here we recover these
627 * bits from the IntlvSel field of the DRAM Limit register (section
628 * 3.4.4.2) for the node that input_addr is associated with.
631 node_id = pvt->mc_node_id;
635 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
636 if (intlv_shift == 0) {
637 debugf1(" InputAddr 0x%lx translates to DramAddr of "
638 "same value\n", (unsigned long)input_addr);
643 bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) +
644 (input_addr & 0xfff);
646 intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1);
647 dram_addr = bits + (intlv_sel << 12);
649 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
650 "(%d node interleave bits)\n", (unsigned long)input_addr,
651 (unsigned long)dram_addr, intlv_shift);
657 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
658 * @dram_addr to a SysAddr.
660 static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
662 struct amd64_pvt *pvt = mci->pvt_info;
663 u64 hole_base, hole_offset, hole_size, base, sys_addr;
666 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
669 if ((dram_addr >= hole_base) &&
670 (dram_addr < (hole_base + hole_size))) {
671 sys_addr = dram_addr + hole_offset;
673 debugf1("using DHAR to translate DramAddr 0x%lx to "
674 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
675 (unsigned long)sys_addr);
681 base = get_dram_base(pvt, pvt->mc_node_id);
682 sys_addr = dram_addr + base;
685 * The sys_addr we have computed up to this point is a 40-bit value
686 * because the k8 deals with 40-bit values. However, the value we are
687 * supposed to return is a full 64-bit physical address. The AMD
688 * x86-64 architecture specifies that the most significant implemented
689 * address bit through bit 63 of a physical address must be either all
690 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
691 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
692 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
695 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
697 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
698 pvt->mc_node_id, (unsigned long)dram_addr,
699 (unsigned long)sys_addr);
705 * @input_addr is an InputAddr associated with the node given by mci. Translate
706 * @input_addr to a SysAddr.
708 static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
711 return dram_addr_to_sys_addr(mci,
712 input_addr_to_dram_addr(mci, input_addr));
716 * Find the minimum and maximum InputAddr values that map to the given @csrow.
717 * Pass back these values in *input_addr_min and *input_addr_max.
719 static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
720 u64 *input_addr_min, u64 *input_addr_max)
722 struct amd64_pvt *pvt;
726 BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt));
728 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
730 *input_addr_min = base & ~mask;
731 *input_addr_max = base | mask;
734 /* Map the Error address to a PAGE and PAGE OFFSET. */
735 static inline void error_address_to_page_and_offset(u64 error_address,
736 u32 *page, u32 *offset)
738 *page = (u32) (error_address >> PAGE_SHIFT);
739 *offset = ((u32) error_address) & ~PAGE_MASK;
743 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
744 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
745 * of a node that detected an ECC memory error. mci represents the node that
746 * the error address maps to (possibly different from the node that detected
747 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
750 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
754 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
757 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
758 "address 0x%lx\n", (unsigned long)sys_addr);
762 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
765 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
768 static unsigned long amd64_determine_edac_cap(struct amd64_pvt *pvt)
771 unsigned long edac_cap = EDAC_FLAG_NONE;
773 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
777 if (pvt->dclr0 & BIT(bit))
778 edac_cap = EDAC_FLAG_SECDED;
783 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
785 static void amd64_dump_dramcfg_low(u32 dclr, int chan)
787 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
789 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
790 (dclr & BIT(16)) ? "un" : "",
791 (dclr & BIT(19)) ? "yes" : "no");
793 debugf1(" PAR/ERR parity: %s\n",
794 (dclr & BIT(8)) ? "enabled" : "disabled");
796 if (boot_cpu_data.x86 == 0x10)
797 debugf1(" DCT 128bit mode width: %s\n",
798 (dclr & BIT(11)) ? "128b" : "64b");
800 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
801 (dclr & BIT(12)) ? "yes" : "no",
802 (dclr & BIT(13)) ? "yes" : "no",
803 (dclr & BIT(14)) ? "yes" : "no",
804 (dclr & BIT(15)) ? "yes" : "no");
807 /* Display and decode various NB registers for debug purposes. */
808 static void dump_misc_regs(struct amd64_pvt *pvt)
810 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
812 debugf1(" NB two channel DRAM capable: %s\n",
813 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
815 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
816 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
817 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
819 amd64_dump_dramcfg_low(pvt->dclr0, 0);
821 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
823 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
825 pvt->dhar, dhar_base(pvt),
826 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
827 : f10_dhar_offset(pvt));
829 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
831 amd64_debug_display_dimm_sizes(pvt, 0);
833 /* everything below this point is Fam10h and above */
834 if (boot_cpu_data.x86 == 0xf)
837 amd64_debug_display_dimm_sizes(pvt, 1);
839 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
841 /* Only if NOT ganged does dclr1 have valid info */
842 if (!dct_ganging_enabled(pvt))
843 amd64_dump_dramcfg_low(pvt->dclr1, 1);
847 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
849 static void prep_chip_selects(struct amd64_pvt *pvt)
851 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
852 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
853 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
855 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
856 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
861 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
863 static void read_dct_base_mask(struct amd64_pvt *pvt)
867 prep_chip_selects(pvt);
869 for_each_chip_select(cs, 0, pvt) {
870 int reg0 = DCSB0 + (cs * 4);
871 int reg1 = DCSB1 + (cs * 4);
872 u32 *base0 = &pvt->csels[0].csbases[cs];
873 u32 *base1 = &pvt->csels[1].csbases[cs];
875 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
876 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
879 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
882 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
883 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
887 for_each_chip_select_mask(cs, 0, pvt) {
888 int reg0 = DCSM0 + (cs * 4);
889 int reg1 = DCSM1 + (cs * 4);
890 u32 *mask0 = &pvt->csels[0].csmasks[cs];
891 u32 *mask1 = &pvt->csels[1].csmasks[cs];
893 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
894 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
897 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
900 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
901 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
906 static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
910 /* F15h supports only DDR3 */
911 if (boot_cpu_data.x86 >= 0x15)
912 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
913 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
914 if (pvt->dchr0 & DDR3_MODE)
915 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
917 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
919 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
922 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
927 /* Get the number of DCT channels the memory controller is using. */
928 static int k8_early_channel_count(struct amd64_pvt *pvt)
932 if (pvt->ext_model >= K8_REV_F)
933 /* RevF (NPT) and later */
934 flag = pvt->dclr0 & WIDTH_128;
936 /* RevE and earlier */
937 flag = pvt->dclr0 & REVE_WIDTH_128;
942 return (flag) ? 2 : 1;
945 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
946 static u64 get_error_address(struct mce *m)
948 struct cpuinfo_x86 *c = &boot_cpu_data;
958 addr = m->addr & GENMASK(start_bit, end_bit);
961 * Erratum 637 workaround
963 if (c->x86 == 0x15) {
964 struct amd64_pvt *pvt;
965 u64 cc6_base, tmp_addr;
967 u8 mce_nid, intlv_en;
969 if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7)
972 mce_nid = amd_get_nb_id(m->extcpu);
973 pvt = mcis[mce_nid]->pvt_info;
975 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
976 intlv_en = tmp >> 21 & 0x7;
978 /* add [47:27] + 3 trailing bits */
979 cc6_base = (tmp & GENMASK(0, 20)) << 3;
981 /* reverse and add DramIntlvEn */
982 cc6_base |= intlv_en ^ 0x7;
988 return cc6_base | (addr & GENMASK(0, 23));
990 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
993 tmp_addr = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1);
995 /* OR DramIntlvSel into bits [14:12] */
996 tmp_addr |= (tmp & GENMASK(21, 23)) >> 9;
998 /* add remaining [11:0] bits from original MC4_ADDR */
999 tmp_addr |= addr & GENMASK(0, 11);
1001 return cc6_base | tmp_addr;
1007 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1009 struct cpuinfo_x86 *c = &boot_cpu_data;
1010 int off = range << 3;
1012 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1013 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1018 if (!dram_rw(pvt, range))
1021 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1022 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1024 /* Factor in CC6 save area by reading dst node's limit reg */
1025 if (c->x86 == 0x15) {
1026 struct pci_dev *f1 = NULL;
1027 u8 nid = dram_dst_node(pvt, range);
1030 f1 = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0x18 + nid, 1));
1034 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1036 pvt->ranges[range].lim.lo &= GENMASK(0, 15);
1038 /* {[39:27],111b} */
1039 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1041 pvt->ranges[range].lim.hi &= GENMASK(0, 7);
1044 pvt->ranges[range].lim.hi |= llim >> 13;
1050 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1053 struct mem_ctl_info *src_mci;
1054 struct amd64_pvt *pvt = mci->pvt_info;
1058 /* CHIPKILL enabled */
1059 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1060 channel = get_channel_from_ecc_syndrome(mci, syndrome);
1063 * Syndrome didn't map, so we don't know which of the
1064 * 2 DIMMs is in error. So we need to ID 'both' of them
1067 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
1068 "error reporting race\n", syndrome);
1069 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1074 * non-chipkill ecc mode
1076 * The k8 documentation is unclear about how to determine the
1077 * channel number when using non-chipkill memory. This method
1078 * was obtained from email communication with someone at AMD.
1079 * (Wish the email was placed in this comment - norsk)
1081 channel = ((sys_addr & BIT(3)) != 0);
1085 * Find out which node the error address belongs to. This may be
1086 * different from the node that detected the error.
1088 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1090 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1091 (unsigned long)sys_addr);
1092 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1096 /* Now map the sys_addr to a CSROW */
1097 csrow = sys_addr_to_csrow(src_mci, sys_addr);
1099 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1101 error_address_to_page_and_offset(sys_addr, &page, &offset);
1103 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1104 channel, EDAC_MOD_STR);
1108 static int ddr2_cs_size(unsigned i, bool dct_width)
1114 else if (!(i & 0x1))
1117 shift = (i + 1) >> 1;
1119 return 128 << (shift + !!dct_width);
1122 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1125 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1127 if (pvt->ext_model >= K8_REV_F) {
1128 WARN_ON(cs_mode > 11);
1129 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1131 else if (pvt->ext_model >= K8_REV_D) {
1132 WARN_ON(cs_mode > 10);
1134 if (cs_mode == 3 || cs_mode == 8)
1135 return 32 << (cs_mode - 1);
1137 return 32 << cs_mode;
1140 WARN_ON(cs_mode > 6);
1141 return 32 << cs_mode;
1146 * Get the number of DCT channels in use.
1149 * number of Memory Channels in operation
1151 * contents of the DCL0_LOW register
1153 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1155 int i, j, channels = 0;
1157 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1158 if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
1162 * Need to check if in unganged mode: In such, there are 2 channels,
1163 * but they are not in 128 bit mode and thus the above 'dclr0' status
1166 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1167 * their CSEnable bit on. If so, then SINGLE DIMM case.
1169 debugf0("Data width is not 128 bits - need more decoding\n");
1172 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1173 * is more than just one DIMM present in unganged mode. Need to check
1174 * both controllers since DIMMs can be placed in either one.
1176 for (i = 0; i < 2; i++) {
1177 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1179 for (j = 0; j < 4; j++) {
1180 if (DBAM_DIMM(j, dbam) > 0) {
1190 amd64_info("MCT channel count: %d\n", channels);
1195 static int ddr3_cs_size(unsigned i, bool dct_width)
1200 if (i == 0 || i == 3 || i == 4)
1206 else if (!(i & 0x1))
1209 shift = (i + 1) >> 1;
1212 cs_size = (128 * (1 << !!dct_width)) << shift;
1217 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1220 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1222 WARN_ON(cs_mode > 11);
1224 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1225 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1227 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1231 * F15h supports only 64bit DCT interfaces
1233 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1236 WARN_ON(cs_mode > 12);
1238 return ddr3_cs_size(cs_mode, false);
1241 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1244 if (boot_cpu_data.x86 == 0xf)
1247 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1248 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1249 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1251 debugf0(" DCTs operate in %s mode.\n",
1252 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1254 if (!dct_ganging_enabled(pvt))
1255 debugf0(" Address range split per DCT: %s\n",
1256 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1258 debugf0(" data interleave for ECC: %s, "
1259 "DRAM cleared since last warm reset: %s\n",
1260 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1261 (dct_memory_cleared(pvt) ? "yes" : "no"));
1263 debugf0(" channel interleave: %s, "
1264 "interleave bits selector: 0x%x\n",
1265 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1266 dct_sel_interleave_addr(pvt));
1269 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
1273 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1274 * Interleaving Modes.
1276 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1277 bool hi_range_sel, u8 intlv_en)
1279 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1281 if (dct_ganging_enabled(pvt))
1285 return dct_sel_high;
1288 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1290 if (dct_interleave_enabled(pvt)) {
1291 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1293 /* return DCT select function: 0=DCT0, 1=DCT1 */
1295 return sys_addr >> 6 & 1;
1297 if (intlv_addr & 0x2) {
1298 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1299 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1301 return ((sys_addr >> shift) & 1) ^ temp;
1304 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1307 if (dct_high_range_enabled(pvt))
1308 return ~dct_sel_high & 1;
1313 /* Convert the sys_addr to the normalized DCT address */
1314 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, unsigned range,
1315 u64 sys_addr, bool hi_rng,
1316 u32 dct_sel_base_addr)
1319 u64 dram_base = get_dram_base(pvt, range);
1320 u64 hole_off = f10_dhar_offset(pvt);
1321 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1326 * base address of high range is below 4Gb
1327 * (bits [47:27] at [31:11])
1328 * DRAM address space on this DCT is hoisted above 4Gb &&
1331 * remove hole offset from sys_addr
1333 * remove high range offset from sys_addr
1335 if ((!(dct_sel_base_addr >> 16) ||
1336 dct_sel_base_addr < dhar_base(pvt)) &&
1338 (sys_addr >= BIT_64(32)))
1339 chan_off = hole_off;
1341 chan_off = dct_sel_base_off;
1345 * we have a valid hole &&
1350 * remove dram base to normalize to DCT address
1352 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1353 chan_off = hole_off;
1355 chan_off = dram_base;
1358 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
1362 * checks if the csrow passed in is marked as SPARED, if so returns the new
1365 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1369 if (online_spare_swap_done(pvt, dct) &&
1370 csrow == online_spare_bad_dramcs(pvt, dct)) {
1372 for_each_chip_select(tmp_cs, dct, pvt) {
1373 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1383 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1384 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1387 * -EINVAL: NOT FOUND
1388 * 0..csrow = Chip-Select Row
1390 static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct)
1392 struct mem_ctl_info *mci;
1393 struct amd64_pvt *pvt;
1394 u64 cs_base, cs_mask;
1395 int cs_found = -EINVAL;
1402 pvt = mci->pvt_info;
1404 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1406 for_each_chip_select(csrow, dct, pvt) {
1407 if (!csrow_enabled(csrow, dct, pvt))
1410 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1412 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1413 csrow, cs_base, cs_mask);
1417 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1418 "(CSBase & ~CSMask)=0x%llx\n",
1419 (in_addr & cs_mask), (cs_base & cs_mask));
1421 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1422 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1424 debugf1(" MATCH csrow=%d\n", cs_found);
1432 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1433 * swapped with a region located at the bottom of memory so that the GPU can use
1434 * the interleaved region and thus two channels.
1436 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1438 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1440 if (boot_cpu_data.x86 == 0x10) {
1441 /* only revC3 and revE have that feature */
1442 if (boot_cpu_data.x86_model < 4 ||
1443 (boot_cpu_data.x86_model < 0xa &&
1444 boot_cpu_data.x86_mask < 3))
1448 amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1450 if (!(swap_reg & 0x1))
1453 swap_base = (swap_reg >> 3) & 0x7f;
1454 swap_limit = (swap_reg >> 11) & 0x7f;
1455 rgn_size = (swap_reg >> 20) & 0x7f;
1456 tmp_addr = sys_addr >> 27;
1458 if (!(sys_addr >> 34) &&
1459 (((tmp_addr >= swap_base) &&
1460 (tmp_addr <= swap_limit)) ||
1461 (tmp_addr < rgn_size)))
1462 return sys_addr ^ (u64)swap_base << 27;
1467 /* For a given @dram_range, check if @sys_addr falls within it. */
1468 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1469 u64 sys_addr, int *nid, int *chan_sel)
1471 int cs_found = -EINVAL;
1475 bool high_range = false;
1477 u8 node_id = dram_dst_node(pvt, range);
1478 u8 intlv_en = dram_intlv_en(pvt, range);
1479 u32 intlv_sel = dram_intlv_sel(pvt, range);
1481 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1482 range, sys_addr, get_dram_limit(pvt, range));
1484 if (dhar_valid(pvt) &&
1485 dhar_base(pvt) <= sys_addr &&
1486 sys_addr < BIT_64(32)) {
1487 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1492 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1495 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1497 dct_sel_base = dct_sel_baseaddr(pvt);
1500 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1501 * select between DCT0 and DCT1.
1503 if (dct_high_range_enabled(pvt) &&
1504 !dct_ganging_enabled(pvt) &&
1505 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1508 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1510 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1511 high_range, dct_sel_base);
1513 /* Remove node interleaving, see F1x120 */
1515 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1516 (chan_addr & 0xfff);
1518 /* remove channel interleave */
1519 if (dct_interleave_enabled(pvt) &&
1520 !dct_high_range_enabled(pvt) &&
1521 !dct_ganging_enabled(pvt)) {
1523 if (dct_sel_interleave_addr(pvt) != 1) {
1524 if (dct_sel_interleave_addr(pvt) == 0x3)
1526 chan_addr = ((chan_addr >> 10) << 9) |
1527 (chan_addr & 0x1ff);
1529 /* A[6] or hash 6 */
1530 chan_addr = ((chan_addr >> 7) << 6) |
1534 chan_addr = ((chan_addr >> 13) << 12) |
1535 (chan_addr & 0xfff);
1538 debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr);
1540 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1542 if (cs_found >= 0) {
1544 *chan_sel = channel;
1549 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1550 int *node, int *chan_sel)
1552 int cs_found = -EINVAL;
1555 for (range = 0; range < DRAM_RANGES; range++) {
1557 if (!dram_rw(pvt, range))
1560 if ((get_dram_base(pvt, range) <= sys_addr) &&
1561 (get_dram_limit(pvt, range) >= sys_addr)) {
1563 cs_found = f1x_match_to_this_node(pvt, range,
1574 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1575 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
1577 * The @sys_addr is usually an error address received from the hardware
1580 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1583 struct amd64_pvt *pvt = mci->pvt_info;
1585 int nid, csrow, chan = 0;
1587 csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
1590 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1594 error_address_to_page_and_offset(sys_addr, &page, &offset);
1597 * We need the syndromes for channel detection only when we're
1598 * ganged. Otherwise @chan should already contain the channel at
1601 if (dct_ganging_enabled(pvt))
1602 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1605 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1609 * Channel unknown, report all channels on this CSROW as failed.
1611 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1612 edac_mc_handle_ce(mci, page, offset, syndrome,
1613 csrow, chan, EDAC_MOD_STR);
1617 * debug routine to display the memory sizes of all logical DIMMs and its
1620 static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1622 int dimm, size0, size1, factor = 0;
1623 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1624 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1626 if (boot_cpu_data.x86 == 0xf) {
1627 if (pvt->dclr0 & WIDTH_128)
1630 /* K8 families < revF not supported yet */
1631 if (pvt->ext_model < K8_REV_F)
1637 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
1638 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1639 : pvt->csels[0].csbases;
1641 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
1643 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1645 /* Dump memory sizes for DIMM and its CSROWs */
1646 for (dimm = 0; dimm < 4; dimm++) {
1649 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
1650 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1651 DBAM_DIMM(dimm, dbam));
1654 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
1655 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1656 DBAM_DIMM(dimm, dbam));
1658 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1659 dimm * 2, size0 << factor,
1660 dimm * 2 + 1, size1 << factor);
1664 static struct amd64_family_type amd64_family_types[] = {
1667 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1668 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1670 .early_channel_count = k8_early_channel_count,
1671 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1672 .dbam_to_cs = k8_dbam_to_chip_select,
1673 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
1678 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1679 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1681 .early_channel_count = f1x_early_channel_count,
1682 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1683 .dbam_to_cs = f10_dbam_to_chip_select,
1684 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1689 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1690 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
1692 .early_channel_count = f1x_early_channel_count,
1693 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1694 .dbam_to_cs = f15_dbam_to_chip_select,
1695 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
1700 static struct pci_dev *pci_get_related_function(unsigned int vendor,
1701 unsigned int device,
1702 struct pci_dev *related)
1704 struct pci_dev *dev = NULL;
1706 dev = pci_get_device(vendor, device, dev);
1708 if ((dev->bus->number == related->bus->number) &&
1709 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1711 dev = pci_get_device(vendor, device, dev);
1718 * These are tables of eigenvectors (one per line) which can be used for the
1719 * construction of the syndrome tables. The modified syndrome search algorithm
1720 * uses those to find the symbol in error and thus the DIMM.
1722 * Algorithm courtesy of Ross LaFetra from AMD.
1724 static u16 x4_vectors[] = {
1725 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1726 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1727 0x0001, 0x0002, 0x0004, 0x0008,
1728 0x1013, 0x3032, 0x4044, 0x8088,
1729 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1730 0x4857, 0xc4fe, 0x13cc, 0x3288,
1731 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1732 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1733 0x15c1, 0x2a42, 0x89ac, 0x4758,
1734 0x2b03, 0x1602, 0x4f0c, 0xca08,
1735 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1736 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1737 0x2b87, 0x164e, 0x642c, 0xdc18,
1738 0x40b9, 0x80de, 0x1094, 0x20e8,
1739 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1740 0x11c1, 0x2242, 0x84ac, 0x4c58,
1741 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1742 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1743 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1744 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1745 0x16b3, 0x3d62, 0x4f34, 0x8518,
1746 0x1e2f, 0x391a, 0x5cac, 0xf858,
1747 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1748 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1749 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1750 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1751 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1752 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1753 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1754 0x185d, 0x2ca6, 0x7914, 0x9e28,
1755 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1756 0x4199, 0x82ee, 0x19f4, 0x2e58,
1757 0x4807, 0xc40e, 0x130c, 0x3208,
1758 0x1905, 0x2e0a, 0x5804, 0xac08,
1759 0x213f, 0x132a, 0xadfc, 0x5ba8,
1760 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
1763 static u16 x8_vectors[] = {
1764 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1765 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1766 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1767 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1768 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1769 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1770 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1771 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1772 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1773 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1774 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1775 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1776 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1777 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1778 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1779 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1780 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1781 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1782 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1785 static int decode_syndrome(u16 syndrome, u16 *vectors, unsigned num_vecs,
1788 unsigned int i, err_sym;
1790 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1792 unsigned v_idx = err_sym * v_dim;
1793 unsigned v_end = (err_sym + 1) * v_dim;
1795 /* walk over all 16 bits of the syndrome */
1796 for (i = 1; i < (1U << 16); i <<= 1) {
1798 /* if bit is set in that eigenvector... */
1799 if (v_idx < v_end && vectors[v_idx] & i) {
1800 u16 ev_comp = vectors[v_idx++];
1802 /* ... and bit set in the modified syndrome, */
1812 /* can't get to zero, move to next symbol */
1817 debugf0("syndrome(%x) not found\n", syndrome);
1821 static int map_err_sym_to_channel(int err_sym, int sym_size)
1834 return err_sym >> 4;
1840 /* imaginary bits not in a DIMM */
1842 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1854 return err_sym >> 3;
1860 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1862 struct amd64_pvt *pvt = mci->pvt_info;
1865 if (pvt->ecc_sym_sz == 8)
1866 err_sym = decode_syndrome(syndrome, x8_vectors,
1867 ARRAY_SIZE(x8_vectors),
1869 else if (pvt->ecc_sym_sz == 4)
1870 err_sym = decode_syndrome(syndrome, x4_vectors,
1871 ARRAY_SIZE(x4_vectors),
1874 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
1878 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
1882 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1883 * ADDRESS and process.
1885 static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m)
1887 struct amd64_pvt *pvt = mci->pvt_info;
1891 /* Ensure that the Error Address is VALID */
1892 if (!(m->status & MCI_STATUS_ADDRV)) {
1893 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1894 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1898 sys_addr = get_error_address(m);
1899 syndrome = extract_syndrome(m->status);
1901 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
1903 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome);
1906 /* Handle any Un-correctable Errors (UEs) */
1907 static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m)
1909 struct mem_ctl_info *log_mci, *src_mci = NULL;
1916 if (!(m->status & MCI_STATUS_ADDRV)) {
1917 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
1918 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1922 sys_addr = get_error_address(m);
1925 * Find out which node the error address belongs to. This may be
1926 * different from the node that detected the error.
1928 src_mci = find_mc_by_sys_addr(mci, sys_addr);
1930 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1931 (unsigned long)sys_addr);
1932 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1938 csrow = sys_addr_to_csrow(log_mci, sys_addr);
1940 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1941 (unsigned long)sys_addr);
1942 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1944 error_address_to_page_and_offset(sys_addr, &page, &offset);
1945 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1949 static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
1952 u16 ec = EC(m->status);
1953 u8 xec = XEC(m->status, 0x1f);
1954 u8 ecc_type = (m->status >> 45) & 0x3;
1956 /* Bail early out if this was an 'observed' error */
1957 if (PP(ec) == NBSL_PP_OBS)
1960 /* Do only ECC errors */
1961 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
1965 amd64_handle_ce(mci, m);
1966 else if (ecc_type == 1)
1967 amd64_handle_ue(mci, m);
1970 void amd64_decode_bus_error(int node_id, struct mce *m)
1972 __amd64_decode_bus_error(mcis[node_id], m);
1976 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
1977 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
1979 static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
1981 /* Reserve the ADDRESS MAP Device */
1982 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1984 amd64_err("error address map device not found: "
1985 "vendor %x device 0x%x (broken BIOS?)\n",
1986 PCI_VENDOR_ID_AMD, f1_id);
1990 /* Reserve the MISC Device */
1991 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1993 pci_dev_put(pvt->F1);
1996 amd64_err("error F3 device not found: "
1997 "vendor %x device 0x%x (broken BIOS?)\n",
1998 PCI_VENDOR_ID_AMD, f3_id);
2002 debugf1("F1: %s\n", pci_name(pvt->F1));
2003 debugf1("F2: %s\n", pci_name(pvt->F2));
2004 debugf1("F3: %s\n", pci_name(pvt->F3));
2009 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2011 pci_dev_put(pvt->F1);
2012 pci_dev_put(pvt->F3);
2016 * Retrieve the hardware registers of the memory controller (this includes the
2017 * 'Address Map' and 'Misc' device regs)
2019 static void read_mc_regs(struct amd64_pvt *pvt)
2021 struct cpuinfo_x86 *c = &boot_cpu_data;
2027 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2028 * those are Read-As-Zero
2030 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2031 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
2033 /* check first whether TOP_MEM2 is enabled */
2034 rdmsrl(MSR_K8_SYSCFG, msr_val);
2035 if (msr_val & (1U << 21)) {
2036 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2037 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2039 debugf0(" TOP_MEM2 disabled.\n");
2041 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2043 read_dram_ctl_register(pvt);
2045 for (range = 0; range < DRAM_RANGES; range++) {
2048 /* read settings for this DRAM range */
2049 read_dram_base_limit_regs(pvt, range);
2051 rw = dram_rw(pvt, range);
2055 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2057 get_dram_base(pvt, range),
2058 get_dram_limit(pvt, range));
2060 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2061 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2062 (rw & 0x1) ? "R" : "-",
2063 (rw & 0x2) ? "W" : "-",
2064 dram_intlv_sel(pvt, range),
2065 dram_dst_node(pvt, range));
2068 read_dct_base_mask(pvt);
2070 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2071 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
2073 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2075 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
2076 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
2078 if (!dct_ganging_enabled(pvt)) {
2079 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
2080 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
2083 pvt->ecc_sym_sz = 4;
2085 if (c->x86 >= 0x10) {
2086 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2087 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
2089 /* F10h, revD and later can do x8 ECC too */
2090 if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
2091 pvt->ecc_sym_sz = 8;
2093 dump_misc_regs(pvt);
2097 * NOTE: CPU Revision Dependent code
2100 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2101 * k8 private pointer to -->
2102 * DRAM Bank Address mapping register
2104 * DCL register where dual_channel_active is
2106 * The DBAM register consists of 4 sets of 4 bits each definitions:
2109 * 0-3 CSROWs 0 and 1
2110 * 4-7 CSROWs 2 and 3
2111 * 8-11 CSROWs 4 and 5
2112 * 12-15 CSROWs 6 and 7
2114 * Values range from: 0 to 15
2115 * The meaning of the values depends on CPU revision and dual-channel state,
2116 * see relevant BKDG more info.
2118 * The memory controller provides for total of only 8 CSROWs in its current
2119 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2120 * single channel or two (2) DIMMs in dual channel mode.
2122 * The following code logic collapses the various tables for CSROW based on CPU
2126 * The number of PAGE_SIZE pages on the specified CSROW number it
2130 static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
2132 u32 cs_mode, nr_pages;
2135 * The math on this doesn't look right on the surface because x/2*4 can
2136 * be simplified to x*2 but this expression makes use of the fact that
2137 * it is integral math where 1/2=0. This intermediate value becomes the
2138 * number of bits to shift the DBAM register to extract the proper CSROW
2141 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
2143 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
2146 * If dual channel then double the memory size of single channel.
2147 * Channel count is 1 or 2
2149 nr_pages <<= (pvt->channel_count - 1);
2151 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
2152 debugf0(" nr_pages= %u channel-count = %d\n",
2153 nr_pages, pvt->channel_count);
2159 * Initialize the array of csrow attribute instances, based on the values
2160 * from pci config hardware registers.
2162 static int init_csrows(struct mem_ctl_info *mci)
2164 struct csrow_info *csrow;
2165 struct amd64_pvt *pvt = mci->pvt_info;
2166 u64 input_addr_min, input_addr_max, sys_addr, base, mask;
2170 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2174 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2175 pvt->mc_node_id, val,
2176 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2178 for_each_chip_select(i, 0, pvt) {
2179 csrow = &mci->csrows[i];
2181 if (!csrow_enabled(i, 0, pvt)) {
2182 debugf1("----CSROW %d EMPTY for node %d\n", i,
2187 debugf1("----CSROW %d VALID for MC node %d\n",
2188 i, pvt->mc_node_id);
2191 csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
2192 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2193 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2194 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2195 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2196 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
2198 get_cs_base_and_mask(pvt, i, 0, &base, &mask);
2199 csrow->page_mask = ~mask;
2200 /* 8 bytes of resolution */
2202 csrow->mtype = amd64_determine_memory_type(pvt, i);
2204 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2205 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2206 (unsigned long)input_addr_min,
2207 (unsigned long)input_addr_max);
2208 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2209 (unsigned long)sys_addr, csrow->page_mask);
2210 debugf1(" nr_pages: %u first_page: 0x%lx "
2211 "last_page: 0x%lx\n",
2212 (unsigned)csrow->nr_pages,
2213 csrow->first_page, csrow->last_page);
2216 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2218 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
2220 (pvt->nbcfg & NBCFG_CHIPKILL) ?
2221 EDAC_S4ECD4ED : EDAC_SECDED;
2223 csrow->edac_mode = EDAC_NONE;
2229 /* get all cores on this DCT */
2230 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid)
2234 for_each_online_cpu(cpu)
2235 if (amd_get_nb_id(cpu) == nid)
2236 cpumask_set_cpu(cpu, mask);
2239 /* check MCG_CTL on all the cpus on this node */
2240 static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid)
2246 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2247 amd64_warn("%s: Error allocating mask\n", __func__);
2251 get_cpus_on_this_dct_cpumask(mask, nid);
2253 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2255 for_each_cpu(cpu, mask) {
2256 struct msr *reg = per_cpu_ptr(msrs, cpu);
2257 nbe = reg->l & MSR_MCGCTL_NBE;
2259 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2261 (nbe ? "enabled" : "disabled"));
2269 free_cpumask_var(mask);
2273 static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
2275 cpumask_var_t cmask;
2278 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2279 amd64_warn("%s: error allocating mask\n", __func__);
2283 get_cpus_on_this_dct_cpumask(cmask, nid);
2285 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2287 for_each_cpu(cpu, cmask) {
2289 struct msr *reg = per_cpu_ptr(msrs, cpu);
2292 if (reg->l & MSR_MCGCTL_NBE)
2293 s->flags.nb_mce_enable = 1;
2295 reg->l |= MSR_MCGCTL_NBE;
2298 * Turn off NB MCE reporting only when it was off before
2300 if (!s->flags.nb_mce_enable)
2301 reg->l &= ~MSR_MCGCTL_NBE;
2304 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2306 free_cpumask_var(cmask);
2311 static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2315 u32 value, mask = 0x3; /* UECC/CECC enable */
2317 if (toggle_ecc_err_reporting(s, nid, ON)) {
2318 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2322 amd64_read_pci_cfg(F3, NBCTL, &value);
2324 s->old_nbctl = value & mask;
2325 s->nbctl_valid = true;
2328 amd64_write_pci_cfg(F3, NBCTL, value);
2330 amd64_read_pci_cfg(F3, NBCFG, &value);
2332 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2333 nid, value, !!(value & NBCFG_ECC_ENABLE));
2335 if (!(value & NBCFG_ECC_ENABLE)) {
2336 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
2338 s->flags.nb_ecc_prev = 0;
2340 /* Attempt to turn on DRAM ECC Enable */
2341 value |= NBCFG_ECC_ENABLE;
2342 amd64_write_pci_cfg(F3, NBCFG, value);
2344 amd64_read_pci_cfg(F3, NBCFG, &value);
2346 if (!(value & NBCFG_ECC_ENABLE)) {
2347 amd64_warn("Hardware rejected DRAM ECC enable,"
2348 "check memory DIMM configuration.\n");
2351 amd64_info("Hardware accepted DRAM ECC Enable\n");
2354 s->flags.nb_ecc_prev = 1;
2357 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2358 nid, value, !!(value & NBCFG_ECC_ENABLE));
2363 static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2366 u32 value, mask = 0x3; /* UECC/CECC enable */
2369 if (!s->nbctl_valid)
2372 amd64_read_pci_cfg(F3, NBCTL, &value);
2374 value |= s->old_nbctl;
2376 amd64_write_pci_cfg(F3, NBCTL, value);
2378 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2379 if (!s->flags.nb_ecc_prev) {
2380 amd64_read_pci_cfg(F3, NBCFG, &value);
2381 value &= ~NBCFG_ECC_ENABLE;
2382 amd64_write_pci_cfg(F3, NBCFG, value);
2385 /* restore the NB Enable MCGCTL bit */
2386 if (toggle_ecc_err_reporting(s, nid, OFF))
2387 amd64_warn("Error restoring NB MCGCTL settings!\n");
2391 * EDAC requires that the BIOS have ECC enabled before
2392 * taking over the processing of ECC errors. A command line
2393 * option allows to force-enable hardware ECC later in
2394 * enable_ecc_error_reporting().
2396 static const char *ecc_msg =
2397 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2398 " Either enable ECC checking or force module loading by setting "
2399 "'ecc_enable_override'.\n"
2400 " (Note that use of the override may cause unknown side effects.)\n";
2402 static bool ecc_enabled(struct pci_dev *F3, u8 nid)
2406 bool nb_mce_en = false;
2408 amd64_read_pci_cfg(F3, NBCFG, &value);
2410 ecc_en = !!(value & NBCFG_ECC_ENABLE);
2411 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
2413 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
2415 amd64_notice("NB MCE bank disabled, set MSR "
2416 "0x%08x[4] on node %d to enable.\n",
2417 MSR_IA32_MCG_CTL, nid);
2419 if (!ecc_en || !nb_mce_en) {
2420 amd64_notice("%s", ecc_msg);
2426 struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2427 ARRAY_SIZE(amd64_inj_attrs) +
2430 struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2432 static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
2434 unsigned int i = 0, j = 0;
2436 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2437 sysfs_attrs[i] = amd64_dbg_attrs[i];
2439 if (boot_cpu_data.x86 >= 0x10)
2440 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2441 sysfs_attrs[i] = amd64_inj_attrs[j];
2443 sysfs_attrs[i] = terminator;
2445 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2448 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2449 struct amd64_family_type *fam)
2451 struct amd64_pvt *pvt = mci->pvt_info;
2453 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2454 mci->edac_ctl_cap = EDAC_FLAG_NONE;
2456 if (pvt->nbcap & NBCAP_SECDED)
2457 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2459 if (pvt->nbcap & NBCAP_CHIPKILL)
2460 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2462 mci->edac_cap = amd64_determine_edac_cap(pvt);
2463 mci->mod_name = EDAC_MOD_STR;
2464 mci->mod_ver = EDAC_AMD64_VERSION;
2465 mci->ctl_name = fam->ctl_name;
2466 mci->dev_name = pci_name(pvt->F2);
2467 mci->ctl_page_to_phys = NULL;
2469 /* memory scrubber interface */
2470 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2471 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2475 * returns a pointer to the family descriptor on success, NULL otherwise.
2477 static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
2479 u8 fam = boot_cpu_data.x86;
2480 struct amd64_family_type *fam_type = NULL;
2484 fam_type = &amd64_family_types[K8_CPUS];
2485 pvt->ops = &amd64_family_types[K8_CPUS].ops;
2489 fam_type = &amd64_family_types[F10_CPUS];
2490 pvt->ops = &amd64_family_types[F10_CPUS].ops;
2494 fam_type = &amd64_family_types[F15_CPUS];
2495 pvt->ops = &amd64_family_types[F15_CPUS].ops;
2499 amd64_err("Unsupported family!\n");
2503 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2505 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
2507 (pvt->ext_model >= K8_REV_F ? "revF or later "
2508 : "revE or earlier ")
2509 : ""), pvt->mc_node_id);
2513 static int amd64_init_one_instance(struct pci_dev *F2)
2515 struct amd64_pvt *pvt = NULL;
2516 struct amd64_family_type *fam_type = NULL;
2517 struct mem_ctl_info *mci = NULL;
2519 u8 nid = get_node_id(F2);
2522 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2526 pvt->mc_node_id = nid;
2530 fam_type = amd64_per_family_init(pvt);
2535 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
2542 * We need to determine how many memory channels there are. Then use
2543 * that information for calculating the size of the dynamic instance
2544 * tables in the 'mci' structure.
2547 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2548 if (pvt->channel_count < 0)
2552 mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid);
2556 mci->pvt_info = pvt;
2557 mci->dev = &pvt->F2->dev;
2559 setup_mci_misc_attrs(mci, fam_type);
2561 if (init_csrows(mci))
2562 mci->edac_cap = EDAC_FLAG_NONE;
2564 set_mc_sysfs_attrs(mci);
2567 if (edac_mc_add_mc(mci)) {
2568 debugf1("failed edac_mc_add_mc()\n");
2572 /* register stuff with EDAC MCE */
2573 if (report_gart_errors)
2574 amd_report_gart_errors(true);
2576 amd_register_ecc_decoder(amd64_decode_bus_error);
2580 atomic_inc(&drv_instances);
2588 free_mc_sibling_devs(pvt);
2597 static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
2598 const struct pci_device_id *mc_type)
2600 u8 nid = get_node_id(pdev);
2601 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2602 struct ecc_settings *s;
2605 ret = pci_enable_device(pdev);
2607 debugf0("ret=%d\n", ret);
2612 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2618 if (!ecc_enabled(F3, nid)) {
2621 if (!ecc_enable_override)
2624 amd64_warn("Forcing ECC on!\n");
2626 if (!enable_ecc_error_reporting(s, nid, F3))
2630 ret = amd64_init_one_instance(pdev);
2632 amd64_err("Error probing instance: %d\n", nid);
2633 restore_ecc_error_reporting(s, nid, F3);
2640 ecc_stngs[nid] = NULL;
2646 static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2648 struct mem_ctl_info *mci;
2649 struct amd64_pvt *pvt;
2650 u8 nid = get_node_id(pdev);
2651 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2652 struct ecc_settings *s = ecc_stngs[nid];
2654 /* Remove from EDAC CORE tracking list */
2655 mci = edac_mc_del_mc(&pdev->dev);
2659 pvt = mci->pvt_info;
2661 restore_ecc_error_reporting(s, nid, F3);
2663 free_mc_sibling_devs(pvt);
2665 /* unregister from EDAC MCE */
2666 amd_report_gart_errors(false);
2667 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2669 kfree(ecc_stngs[nid]);
2670 ecc_stngs[nid] = NULL;
2672 /* Free the EDAC CORE resources */
2673 mci->pvt_info = NULL;
2681 * This table is part of the interface for loading drivers for PCI devices. The
2682 * PCI core identifies what devices are on a system during boot, and then
2683 * inquiry this table to see if this driver is for a given device found.
2685 static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2687 .vendor = PCI_VENDOR_ID_AMD,
2688 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2689 .subvendor = PCI_ANY_ID,
2690 .subdevice = PCI_ANY_ID,
2695 .vendor = PCI_VENDOR_ID_AMD,
2696 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2697 .subvendor = PCI_ANY_ID,
2698 .subdevice = PCI_ANY_ID,
2703 .vendor = PCI_VENDOR_ID_AMD,
2704 .device = PCI_DEVICE_ID_AMD_15H_NB_F2,
2705 .subvendor = PCI_ANY_ID,
2706 .subdevice = PCI_ANY_ID,
2713 MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2715 static struct pci_driver amd64_pci_driver = {
2716 .name = EDAC_MOD_STR,
2717 .probe = amd64_probe_one_instance,
2718 .remove = __devexit_p(amd64_remove_one_instance),
2719 .id_table = amd64_pci_table,
2722 static void setup_pci_device(void)
2724 struct mem_ctl_info *mci;
2725 struct amd64_pvt *pvt;
2733 pvt = mci->pvt_info;
2735 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2737 if (!amd64_ctl_pci) {
2738 pr_warning("%s(): Unable to create PCI control\n",
2741 pr_warning("%s(): PCI error report via EDAC not set\n",
2747 static int __init amd64_edac_init(void)
2751 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
2755 if (amd_cache_northbridges() < 0)
2759 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2760 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
2761 if (!(mcis && ecc_stngs))
2764 msrs = msrs_alloc();
2768 err = pci_register_driver(&amd64_pci_driver);
2773 if (!atomic_read(&drv_instances))
2774 goto err_no_instances;
2780 pci_unregister_driver(&amd64_pci_driver);
2797 static void __exit amd64_edac_exit(void)
2800 edac_pci_release_generic_ctl(amd64_ctl_pci);
2802 pci_unregister_driver(&amd64_pci_driver);
2814 module_init(amd64_edac_init);
2815 module_exit(amd64_edac_exit);
2817 MODULE_LICENSE("GPL");
2818 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2819 "Dave Peterson, Thayne Harbaugh");
2820 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2821 EDAC_AMD64_VERSION);
2823 module_param(edac_op_state, int, 0444);
2824 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");