1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Driver for Pondicherry2 memory controller. 4 * 5 * Copyright (c) 2016, Intel Corporation. 6 * 7 * [Derived from sb_edac.c] 8 * 9 * Translation of system physical addresses to DIMM addresses 10 * is a two stage process: 11 * 12 * First the Pondicherry 2 memory controller handles slice and channel interleaving 13 * in "sys2pmi()". This is (almost) completley common between platforms. 14 * 15 * Then a platform specific dunit (DIMM unit) completes the process to provide DIMM, 16 * rank, bank, row and column using the appropriate "dunit_ops" functions/parameters. 17 */ 18 19#include <linux/module.h> 20#include <linux/init.h> 21#include <linux/pci.h> 22#include <linux/pci_ids.h> 23#include <linux/slab.h> 24#include <linux/delay.h> 25#include <linux/edac.h> 26#include <linux/mmzone.h> 27#include <linux/smp.h> 28#include <linux/bitmap.h> 29#include <linux/math64.h> 30#include <linux/mod_devicetable.h> 31#include <linux/platform_data/x86/p2sb.h> 32 33#include <asm/cpu_device_id.h> 34#include <asm/intel-family.h> 35#include <asm/processor.h> 36#include <asm/mce.h> 37 38#include "edac_mc.h" 39#include "edac_module.h" 40#include "pnd2_edac.h" 41 42#define EDAC_MOD_STR "pnd2_edac" 43 44#define APL_NUM_CHANNELS 4 45#define DNV_NUM_CHANNELS 2 46#define DNV_MAX_DIMMS 2 /* Max DIMMs per channel */ 47 48enum type { 49 APL, 50 DNV, /* All requests go to PMI CH0 on each slice (CH1 disabled) */ 51}; 52 53struct dram_addr { 54 int chan; 55 int dimm; 56 int rank; 57 int bank; 58 int row; 59 int col; 60}; 61 62struct pnd2_pvt { 63 int dimm_geom[APL_NUM_CHANNELS]; 64 u64 tolm, tohm; 65}; 66 67/* 68 * System address space is divided into multiple regions with 69 * different interleave rules in each. The as0/as1 regions 70 * have no interleaving at all. The as2 region is interleaved 71 * between two channels. The mot region is magic and may overlap 72 * other regions, with its interleave rules taking precedence. 73 * Addresses not in any of these regions are interleaved across 74 * all four channels. 75 */ 76static struct region { 77 u64 base; 78 u64 limit; 79 u8 enabled; 80} mot, as0, as1, as2; 81 82static struct dunit_ops { 83 char *name; 84 enum type type; 85 int pmiaddr_shift; 86 int pmiidx_shift; 87 int channels; 88 int dimms_per_channel; 89 int (*rd_reg)(int port, int off, int op, void *data, size_t sz, char *name); 90 int (*get_registers)(void); 91 int (*check_ecc)(void); 92 void (*mk_region)(char *name, struct region *rp, void *asym); 93 void (*get_dimm_config)(struct mem_ctl_info *mci); 94 int (*pmi2mem)(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx, 95 struct dram_addr *daddr, char *msg); 96} *ops; 97 98static struct mem_ctl_info *pnd2_mci; 99 100#define PND2_MSG_SIZE 256 101 102/* Debug macros */ 103#define pnd2_printk(level, fmt, arg...) \ 104 edac_printk(level, "pnd2", fmt, ##arg) 105 106#define pnd2_mc_printk(mci, level, fmt, arg...) \ 107 edac_mc_chipset_printk(mci, level, "pnd2", fmt, ##arg) 108 109#define MOT_CHAN_INTLV_BIT_1SLC_2CH 12 110#define MOT_CHAN_INTLV_BIT_2SLC_2CH 13 111#define SELECTOR_DISABLED (-1) 112#define _4GB (1ul << 32) 113 114#define PMI_ADDRESS_WIDTH 31 115#define PND_MAX_PHYS_BIT 39 116 117#define APL_ASYMSHIFT 28 118#define DNV_ASYMSHIFT 31 119#define CH_HASH_MASK_LSB 6 120#define SLICE_HASH_MASK_LSB 6 121#define MOT_SLC_INTLV_BIT 12 122#define LOG2_PMI_ADDR_GRANULARITY 5 123#define MOT_SHIFT 24 124 125#define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo)) 126#define U64_LSHIFT(val, s) ((u64)(val) << (s)) 127 128/* 129 * On Apollo Lake we access memory controller registers via a 130 * side-band mailbox style interface in a hidden PCI device 131 * configuration space. 132 */ 133static struct pci_bus *p2sb_bus; 134#define P2SB_DEVFN PCI_DEVFN(0xd, 0) 135#define P2SB_ADDR_OFF 0xd0 136#define P2SB_DATA_OFF 0xd4 137#define P2SB_STAT_OFF 0xd8 138#define P2SB_ROUT_OFF 0xda 139#define P2SB_EADD_OFF 0xdc 140#define P2SB_HIDE_OFF 0xe1 141 142#define P2SB_BUSY 1 143 144#define P2SB_READ(size, off, ptr) \ 145 pci_bus_read_config_##size(p2sb_bus, P2SB_DEVFN, off, ptr) 146#define P2SB_WRITE(size, off, val) \ 147 pci_bus_write_config_##size(p2sb_bus, P2SB_DEVFN, off, val) 148 149static bool p2sb_is_busy(u16 *status) 150{ 151 P2SB_READ(word, P2SB_STAT_OFF, status); 152 153 return !!(*status & P2SB_BUSY); 154} 155 156static int _apl_rd_reg(int port, int off, int op, u32 *data) 157{ 158 int retries = 0xff, ret; 159 u16 status; 160 u8 hidden; 161 162 /* Unhide the P2SB device, if it's hidden */ 163 P2SB_READ(byte, P2SB_HIDE_OFF, &hidden); 164 if (hidden) 165 P2SB_WRITE(byte, P2SB_HIDE_OFF, 0); 166 167 if (p2sb_is_busy(&status)) { 168 ret = -EAGAIN; 169 goto out; 170 } 171 172 P2SB_WRITE(dword, P2SB_ADDR_OFF, (port << 24) | off); 173 P2SB_WRITE(dword, P2SB_DATA_OFF, 0); 174 P2SB_WRITE(dword, P2SB_EADD_OFF, 0); 175 P2SB_WRITE(word, P2SB_ROUT_OFF, 0); 176 P2SB_WRITE(word, P2SB_STAT_OFF, (op << 8) | P2SB_BUSY); 177 178 while (p2sb_is_busy(&status)) { 179 if (retries-- == 0) { 180 ret = -EBUSY; 181 goto out; 182 } 183 } 184 185 P2SB_READ(dword, P2SB_DATA_OFF, data); 186 ret = (status >> 1) & 0x3; 187out: 188 /* Hide the P2SB device, if it was hidden before */ 189 if (hidden) 190 P2SB_WRITE(byte, P2SB_HIDE_OFF, hidden); 191 192 return ret; 193} 194 195static int apl_rd_reg(int port, int off, int op, void *data, size_t sz, char *name) 196{ 197 int ret = 0; 198 199 edac_dbg(2, "Read %s port=%x off=%x op=%x\n", name, port, off, op); 200 switch (sz) { 201 case 8: 202 ret = _apl_rd_reg(port, off + 4, op, (u32 *)(data + 4)); 203 fallthrough; 204 case 4: 205 ret |= _apl_rd_reg(port, off, op, (u32 *)data); 206 pnd2_printk(KERN_DEBUG, "%s=%x%08x ret=%d\n", name, 207 sz == 8 ? *((u32 *)(data + 4)) : 0, *((u32 *)data), ret); 208 break; 209 } 210 211 return ret; 212} 213 214static u64 get_mem_ctrl_hub_base_addr(void) 215{ 216 struct b_cr_mchbar_lo_pci lo; 217 struct b_cr_mchbar_hi_pci hi; 218 struct pci_dev *pdev; 219 220 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x1980, NULL); 221 if (pdev) { 222 pci_read_config_dword(pdev, 0x48, (u32 *)&lo); 223 pci_read_config_dword(pdev, 0x4c, (u32 *)&hi); 224 pci_dev_put(pdev); 225 } else { 226 return 0; 227 } 228 229 if (!lo.enable) { 230 edac_dbg(2, "MMIO via memory controller hub base address is disabled!\n"); 231 return 0; 232 } 233 234 return U64_LSHIFT(hi.base, 32) | U64_LSHIFT(lo.base, 15); 235} 236 237#define DNV_MCHBAR_SIZE 0x8000 238#define DNV_SB_PORT_SIZE 0x10000 239static int dnv_rd_reg(int port, int off, int op, void *data, size_t sz, char *name) 240{ 241 struct pci_dev *pdev; 242 void __iomem *base; 243 struct resource r; 244 int ret; 245 246 if (op == 4) { 247 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x1980, NULL); 248 if (!pdev) 249 return -ENODEV; 250 251 pci_read_config_dword(pdev, off, data); 252 pci_dev_put(pdev); 253 } else { 254 /* MMIO via memory controller hub base address */ 255 if (op == 0 && port == 0x4c) { 256 memset(&r, 0, sizeof(r)); 257 258 r.start = get_mem_ctrl_hub_base_addr(); 259 if (!r.start) 260 return -ENODEV; 261 r.end = r.start + DNV_MCHBAR_SIZE - 1; 262 } else { 263 /* MMIO via sideband register base address */ 264 ret = p2sb_bar(NULL, 0, &r); 265 if (ret) 266 return ret; 267 268 r.start += (port << 16); 269 r.end = r.start + DNV_SB_PORT_SIZE - 1; 270 } 271 272 base = ioremap(r.start, resource_size(&r)); 273 if (!base) 274 return -ENODEV; 275 276 if (sz == 8) 277 *(u64 *)data = readq(base + off); 278 else 279 *(u32 *)data = readl(base + off); 280 281 iounmap(base); 282 } 283 284 edac_dbg(2, "Read %s=%.8x_%.8x\n", name, 285 (sz == 8) ? *(u32 *)(data + 4) : 0, *(u32 *)data); 286 287 return 0; 288} 289 290#define RD_REGP(regp, regname, port) \ 291 ops->rd_reg(port, \ 292 regname##_offset, \ 293 regname##_r_opcode, \ 294 regp, sizeof(struct regname), \ 295 #regname) 296 297#define RD_REG(regp, regname) \ 298 ops->rd_reg(regname ## _port, \ 299 regname##_offset, \ 300 regname##_r_opcode, \ 301 regp, sizeof(struct regname), \ 302 #regname) 303 304static u64 top_lm, top_hm; 305static bool two_slices; 306static bool two_channels; /* Both PMI channels in one slice enabled */ 307 308static u8 sym_chan_mask; 309static u8 asym_chan_mask; 310static u8 chan_mask; 311 312static int slice_selector = -1; 313static int chan_selector = -1; 314static u64 slice_hash_mask; 315static u64 chan_hash_mask; 316 317static void mk_region(char *name, struct region *rp, u64 base, u64 limit) 318{ 319 rp->enabled = 1; 320 rp->base = base; 321 rp->limit = limit; 322 edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, limit); 323} 324 325static void mk_region_mask(char *name, struct region *rp, u64 base, u64 mask) 326{ 327 if (mask == 0) { 328 pr_info(FW_BUG "MOT mask cannot be zero\n"); 329 return; 330 } 331 if (mask != GENMASK_ULL(PND_MAX_PHYS_BIT, __ffs(mask))) { 332 pr_info(FW_BUG "MOT mask not power of two\n"); 333 return; 334 } 335 if (base & ~mask) { 336 pr_info(FW_BUG "MOT region base/mask alignment error\n"); 337 return; 338 } 339 rp->base = base; 340 rp->limit = (base | ~mask) & GENMASK_ULL(PND_MAX_PHYS_BIT, 0); 341 rp->enabled = 1; 342 edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, rp->limit); 343} 344 345static bool in_region(struct region *rp, u64 addr) 346{ 347 if (!rp->enabled) 348 return false; 349 350 return rp->base <= addr && addr <= rp->limit; 351} 352 353static int gen_sym_mask(struct b_cr_slice_channel_hash *p) 354{ 355 int mask = 0; 356 357 if (!p->slice_0_mem_disabled) 358 mask |= p->sym_slice0_channel_enabled; 359 360 if (!p->slice_1_disabled) 361 mask |= p->sym_slice1_channel_enabled << 2; 362 363 if (p->ch_1_disabled || p->enable_pmi_dual_data_mode) 364 mask &= 0x5; 365 366 return mask; 367} 368 369static int gen_asym_mask(struct b_cr_slice_channel_hash *p, 370 struct b_cr_asym_mem_region0_mchbar *as0, 371 struct b_cr_asym_mem_region1_mchbar *as1, 372 struct b_cr_asym_2way_mem_region_mchbar *as2way) 373{ 374 const int intlv[] = { 0x5, 0xA, 0x3, 0xC }; 375 int mask = 0; 376 377 if (as2way->asym_2way_interleave_enable) 378 mask = intlv[as2way->asym_2way_intlv_mode]; 379 if (as0->slice0_asym_enable) 380 mask |= (1 << as0->slice0_asym_channel_select); 381 if (as1->slice1_asym_enable) 382 mask |= (4 << as1->slice1_asym_channel_select); 383 if (p->slice_0_mem_disabled) 384 mask &= 0xc; 385 if (p->slice_1_disabled) 386 mask &= 0x3; 387 if (p->ch_1_disabled || p->enable_pmi_dual_data_mode) 388 mask &= 0x5; 389 390 return mask; 391} 392 393static struct b_cr_tolud_pci tolud; 394static struct b_cr_touud_lo_pci touud_lo; 395static struct b_cr_touud_hi_pci touud_hi; 396static struct b_cr_asym_mem_region0_mchbar asym0; 397static struct b_cr_asym_mem_region1_mchbar asym1; 398static struct b_cr_asym_2way_mem_region_mchbar asym_2way; 399static struct b_cr_mot_out_base_mchbar mot_base; 400static struct b_cr_mot_out_mask_mchbar mot_mask; 401static struct b_cr_slice_channel_hash chash; 402 403/* Apollo Lake dunit */ 404/* 405 * Validated on board with just two DIMMs in the [0] and [2] positions 406 * in this array. Other port number matches documentation, but caution 407 * advised. 408 */ 409static const int apl_dports[APL_NUM_CHANNELS] = { 0x18, 0x10, 0x11, 0x19 }; 410static struct d_cr_drp0 drp0[APL_NUM_CHANNELS]; 411 412/* Denverton dunit */ 413static const int dnv_dports[DNV_NUM_CHANNELS] = { 0x10, 0x12 }; 414static struct d_cr_dsch dsch; 415static struct d_cr_ecc_ctrl ecc_ctrl[DNV_NUM_CHANNELS]; 416static struct d_cr_drp drp[DNV_NUM_CHANNELS]; 417static struct d_cr_dmap dmap[DNV_NUM_CHANNELS]; 418static struct d_cr_dmap1 dmap1[DNV_NUM_CHANNELS]; 419static struct d_cr_dmap2 dmap2[DNV_NUM_CHANNELS]; 420static struct d_cr_dmap3 dmap3[DNV_NUM_CHANNELS]; 421static struct d_cr_dmap4 dmap4[DNV_NUM_CHANNELS]; 422static struct d_cr_dmap5 dmap5[DNV_NUM_CHANNELS]; 423 424static void apl_mk_region(char *name, struct region *rp, void *asym) 425{ 426 struct b_cr_asym_mem_region0_mchbar *a = asym; 427 428 mk_region(name, rp, 429 U64_LSHIFT(a->slice0_asym_base, APL_ASYMSHIFT), 430 U64_LSHIFT(a->slice0_asym_limit, APL_ASYMSHIFT) + 431 GENMASK_ULL(APL_ASYMSHIFT - 1, 0)); 432} 433 434static void dnv_mk_region(char *name, struct region *rp, void *asym) 435{ 436 struct b_cr_asym_mem_region_denverton *a = asym; 437 438 mk_region(name, rp, 439 U64_LSHIFT(a->slice_asym_base, DNV_ASYMSHIFT), 440 U64_LSHIFT(a->slice_asym_limit, DNV_ASYMSHIFT) + 441 GENMASK_ULL(DNV_ASYMSHIFT - 1, 0)); 442} 443 444static int apl_get_registers(void) 445{ 446 int ret = -ENODEV; 447 int i; 448 449 if (RD_REG(&asym_2way, b_cr_asym_2way_mem_region_mchbar)) 450 return -ENODEV; 451 452 /* 453 * RD_REGP() will fail for unpopulated or non-existent 454 * DIMM slots. Return success if we find at least one DIMM. 455 */ 456 for (i = 0; i < APL_NUM_CHANNELS; i++) 457 if (!RD_REGP(&drp0[i], d_cr_drp0, apl_dports[i])) 458 ret = 0; 459 460 return ret; 461} 462 463static int dnv_get_registers(void) 464{ 465 int i; 466 467 if (RD_REG(&dsch, d_cr_dsch)) 468 return -ENODEV; 469 470 for (i = 0; i < DNV_NUM_CHANNELS; i++) 471 if (RD_REGP(&ecc_ctrl[i], d_cr_ecc_ctrl, dnv_dports[i]) || 472 RD_REGP(&drp[i], d_cr_drp, dnv_dports[i]) || 473 RD_REGP(&dmap[i], d_cr_dmap, dnv_dports[i]) || 474 RD_REGP(&dmap1[i], d_cr_dmap1, dnv_dports[i]) || 475 RD_REGP(&dmap2[i], d_cr_dmap2, dnv_dports[i]) || 476 RD_REGP(&dmap3[i], d_cr_dmap3, dnv_dports[i]) || 477 RD_REGP(&dmap4[i], d_cr_dmap4, dnv_dports[i]) || 478 RD_REGP(&dmap5[i], d_cr_dmap5, dnv_dports[i])) 479 return -ENODEV; 480 481 return 0; 482} 483 484/* 485 * Read all the h/w config registers once here (they don't 486 * change at run time. Figure out which address ranges have 487 * which interleave characteristics. 488 */ 489static int get_registers(void) 490{ 491 const int intlv[] = { 10, 11, 12, 12 }; 492 493 if (RD_REG(&tolud, b_cr_tolud_pci) || 494 RD_REG(&touud_lo, b_cr_touud_lo_pci) || 495 RD_REG(&touud_hi, b_cr_touud_hi_pci) || 496 RD_REG(&asym0, b_cr_asym_mem_region0_mchbar) || 497 RD_REG(&asym1, b_cr_asym_mem_region1_mchbar) || 498 RD_REG(&mot_base, b_cr_mot_out_base_mchbar) || 499 RD_REG(&mot_mask, b_cr_mot_out_mask_mchbar) || 500 RD_REG(&chash, b_cr_slice_channel_hash)) 501 return -ENODEV; 502 503 if (ops->get_registers()) 504 return -ENODEV; 505 506 if (ops->type == DNV) { 507 /* PMI channel idx (always 0) for asymmetric region */ 508 asym0.slice0_asym_channel_select = 0; 509 asym1.slice1_asym_channel_select = 0; 510 /* PMI channel bitmap (always 1) for symmetric region */ 511 chash.sym_slice0_channel_enabled = 0x1; 512 chash.sym_slice1_channel_enabled = 0x1; 513 } 514 515 if (asym0.slice0_asym_enable) 516 ops->mk_region("as0", &as0, &asym0); 517 518 if (asym1.slice1_asym_enable) 519 ops->mk_region("as1", &as1, &asym1); 520 521 if (asym_2way.asym_2way_interleave_enable) { 522 mk_region("as2way", &as2, 523 U64_LSHIFT(asym_2way.asym_2way_base, APL_ASYMSHIFT), 524 U64_LSHIFT(asym_2way.asym_2way_limit, APL_ASYMSHIFT) + 525 GENMASK_ULL(APL_ASYMSHIFT - 1, 0)); 526 } 527 528 if (mot_base.imr_en) { 529 mk_region_mask("mot", &mot, 530 U64_LSHIFT(mot_base.mot_out_base, MOT_SHIFT), 531 U64_LSHIFT(mot_mask.mot_out_mask, MOT_SHIFT)); 532 } 533 534 top_lm = U64_LSHIFT(tolud.tolud, 20); 535 top_hm = U64_LSHIFT(touud_hi.touud, 32) | U64_LSHIFT(touud_lo.touud, 20); 536 537 two_slices = !chash.slice_1_disabled && 538 !chash.slice_0_mem_disabled && 539 (chash.sym_slice0_channel_enabled != 0) && 540 (chash.sym_slice1_channel_enabled != 0); 541 two_channels = !chash.ch_1_disabled && 542 !chash.enable_pmi_dual_data_mode && 543 ((chash.sym_slice0_channel_enabled == 3) || 544 (chash.sym_slice1_channel_enabled == 3)); 545 546 sym_chan_mask = gen_sym_mask(&chash); 547 asym_chan_mask = gen_asym_mask(&chash, &asym0, &asym1, &asym_2way); 548 chan_mask = sym_chan_mask | asym_chan_mask; 549 550 if (two_slices && !two_channels) { 551 if (chash.hvm_mode) 552 slice_selector = 29; 553 else 554 slice_selector = intlv[chash.interleave_mode]; 555 } else if (!two_slices && two_channels) { 556 if (chash.hvm_mode) 557 chan_selector = 29; 558 else 559 chan_selector = intlv[chash.interleave_mode]; 560 } else if (two_slices && two_channels) { 561 if (chash.hvm_mode) { 562 slice_selector = 29; 563 chan_selector = 30; 564 } else { 565 slice_selector = intlv[chash.interleave_mode]; 566 chan_selector = intlv[chash.interleave_mode] + 1; 567 } 568 } 569 570 if (two_slices) { 571 if (!chash.hvm_mode) 572 slice_hash_mask = chash.slice_hash_mask << SLICE_HASH_MASK_LSB; 573 if (!two_channels) 574 slice_hash_mask |= BIT_ULL(slice_selector); 575 } 576 577 if (two_channels) { 578 if (!chash.hvm_mode) 579 chan_hash_mask = chash.ch_hash_mask << CH_HASH_MASK_LSB; 580 if (!two_slices) 581 chan_hash_mask |= BIT_ULL(chan_selector); 582 } 583 584 return 0; 585} 586 587/* Get a contiguous memory address (remove the MMIO gap) */ 588static u64 remove_mmio_gap(u64 sys) 589{ 590 return (sys < _4GB) ? sys : sys - (_4GB - top_lm); 591} 592 593/* Squeeze out one address bit, shift upper part down to fill gap */ 594static void remove_addr_bit(u64 *addr, int bitidx) 595{ 596 u64 mask; 597 598 if (bitidx == -1) 599 return; 600 601 mask = (1ull << bitidx) - 1; 602 *addr = ((*addr >> 1) & ~mask) | (*addr & mask); 603} 604 605/* XOR all the bits from addr specified in mask */ 606static int hash_by_mask(u64 addr, u64 mask) 607{ 608 u64 result = addr & mask; 609 610 result = (result >> 32) ^ result; 611 result = (result >> 16) ^ result; 612 result = (result >> 8) ^ result; 613 result = (result >> 4) ^ result; 614 result = (result >> 2) ^ result; 615 result = (result >> 1) ^ result; 616 617 return (int)result & 1; 618} 619 620/* 621 * First stage decode. Take the system address and figure out which 622 * second stage will deal with it based on interleave modes. 623 */ 624static int sys2pmi(const u64 addr, u32 *pmiidx, u64 *pmiaddr, char *msg) 625{ 626 u64 contig_addr, contig_base, contig_offset, contig_base_adj; 627 int mot_intlv_bit = two_slices ? MOT_CHAN_INTLV_BIT_2SLC_2CH : 628 MOT_CHAN_INTLV_BIT_1SLC_2CH; 629 int slice_intlv_bit_rm = SELECTOR_DISABLED; 630 int chan_intlv_bit_rm = SELECTOR_DISABLED; 631 /* Determine if address is in the MOT region. */ 632 bool mot_hit = in_region(&mot, addr); 633 /* Calculate the number of symmetric regions enabled. */ 634 int sym_channels = hweight8(sym_chan_mask); 635 636 /* 637 * The amount we need to shift the asym base can be determined by the 638 * number of enabled symmetric channels. 639 * NOTE: This can only work because symmetric memory is not supposed 640 * to do a 3-way interleave. 641 */ 642 int sym_chan_shift = sym_channels >> 1; 643 644 /* Give up if address is out of range, or in MMIO gap */ 645 if (addr >= (1ul << PND_MAX_PHYS_BIT) || 646 (addr >= top_lm && addr < _4GB) || addr >= top_hm) { 647 snprintf(msg, PND2_MSG_SIZE, "Error address 0x%llx is not DRAM", addr); 648 return -EINVAL; 649 } 650 651 /* Get a contiguous memory address (remove the MMIO gap) */ 652 contig_addr = remove_mmio_gap(addr); 653 654 if (in_region(&as0, addr)) { 655 *pmiidx = asym0.slice0_asym_channel_select; 656 657 contig_base = remove_mmio_gap(as0.base); 658 contig_offset = contig_addr - contig_base; 659 contig_base_adj = (contig_base >> sym_chan_shift) * 660 ((chash.sym_slice0_channel_enabled >> (*pmiidx & 1)) & 1); 661 contig_addr = contig_offset + ((sym_channels > 0) ? contig_base_adj : 0ull); 662 } else if (in_region(&as1, addr)) { 663 *pmiidx = 2u + asym1.slice1_asym_channel_select; 664 665 contig_base = remove_mmio_gap(as1.base); 666 contig_offset = contig_addr - contig_base; 667 contig_base_adj = (contig_base >> sym_chan_shift) * 668 ((chash.sym_slice1_channel_enabled >> (*pmiidx & 1)) & 1); 669 contig_addr = contig_offset + ((sym_channels > 0) ? contig_base_adj : 0ull); 670 } else if (in_region(&as2, addr) && (asym_2way.asym_2way_intlv_mode == 0x3ul)) { 671 bool channel1; 672 673 mot_intlv_bit = MOT_CHAN_INTLV_BIT_1SLC_2CH; 674 *pmiidx = (asym_2way.asym_2way_intlv_mode & 1) << 1; 675 channel1 = mot_hit ? ((bool)((addr >> mot_intlv_bit) & 1)) : 676 hash_by_mask(contig_addr, chan_hash_mask); 677 *pmiidx |= (u32)channel1; 678 679 contig_base = remove_mmio_gap(as2.base); 680 chan_intlv_bit_rm = mot_hit ? mot_intlv_bit : chan_selector; 681 contig_offset = contig_addr - contig_base; 682 remove_addr_bit(&contig_offset, chan_intlv_bit_rm); 683 contig_addr = (contig_base >> sym_chan_shift) + contig_offset; 684 } else { 685 /* Otherwise we're in normal, boring symmetric mode. */ 686 *pmiidx = 0u; 687 688 if (two_slices) { 689 bool slice1; 690 691 if (mot_hit) { 692 slice_intlv_bit_rm = MOT_SLC_INTLV_BIT; 693 slice1 = (addr >> MOT_SLC_INTLV_BIT) & 1; 694 } else { 695 slice_intlv_bit_rm = slice_selector; 696 slice1 = hash_by_mask(addr, slice_hash_mask); 697 } 698 699 *pmiidx = (u32)slice1 << 1; 700 } 701 702 if (two_channels) { 703 bool channel1; 704 705 mot_intlv_bit = two_slices ? MOT_CHAN_INTLV_BIT_2SLC_2CH : 706 MOT_CHAN_INTLV_BIT_1SLC_2CH; 707 708 if (mot_hit) { 709 chan_intlv_bit_rm = mot_intlv_bit; 710 channel1 = (addr >> mot_intlv_bit) & 1; 711 } else { 712 chan_intlv_bit_rm = chan_selector; 713 channel1 = hash_by_mask(contig_addr, chan_hash_mask); 714 } 715 716 *pmiidx |= (u32)channel1; 717 } 718 } 719 720 /* Remove the chan_selector bit first */ 721 remove_addr_bit(&contig_addr, chan_intlv_bit_rm); 722 /* Remove the slice bit (we remove it second because it must be lower */ 723 remove_addr_bit(&contig_addr, slice_intlv_bit_rm); 724 *pmiaddr = contig_addr; 725 726 return 0; 727} 728 729/* Translate PMI address to memory (rank, row, bank, column) */ 730#define C(n) (0x10 | (n)) /* column */ 731#define B(n) (0x20 | (n)) /* bank */ 732#define R(n) (0x40 | (n)) /* row */ 733#define RS (0x80) /* rank */ 734 735/* addrdec values */ 736#define AMAP_1KB 0 737#define AMAP_2KB 1 738#define AMAP_4KB 2 739#define AMAP_RSVD 3 740 741/* dden values */ 742#define DEN_4Gb 0 743#define DEN_8Gb 2 744 745/* dwid values */ 746#define X8 0 747#define X16 1 748 749static struct dimm_geometry { 750 u8 addrdec; 751 u8 dden; 752 u8 dwid; 753 u8 rowbits, colbits; 754 u16 bits[PMI_ADDRESS_WIDTH]; 755} dimms[] = { 756 { 757 .addrdec = AMAP_1KB, .dden = DEN_4Gb, .dwid = X16, 758 .rowbits = 15, .colbits = 10, 759 .bits = { 760 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0), 761 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9), 762 R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14), 763 0, 0, 0, 0 764 } 765 }, 766 { 767 .addrdec = AMAP_1KB, .dden = DEN_4Gb, .dwid = X8, 768 .rowbits = 16, .colbits = 10, 769 .bits = { 770 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0), 771 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9), 772 R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14), 773 R(15), 0, 0, 0 774 } 775 }, 776 { 777 .addrdec = AMAP_1KB, .dden = DEN_8Gb, .dwid = X16, 778 .rowbits = 16, .colbits = 10, 779 .bits = { 780 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0), 781 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9), 782 R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14), 783 R(15), 0, 0, 0 784 } 785 }, 786 { 787 .addrdec = AMAP_1KB, .dden = DEN_8Gb, .dwid = X8, 788 .rowbits = 16, .colbits = 11, 789 .bits = { 790 C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0), 791 R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9), 792 R(10), C(7), C(8), C(9), R(11), RS, C(11), R(12), R(13), 793 R(14), R(15), 0, 0 794 } 795 }, 796 { 797 .addrdec = AMAP_2KB, .dden = DEN_4Gb, .dwid = X16, 798 .rowbits = 15, .colbits = 10, 799 .bits = { 800 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2), 801 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), 802 R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14), 803 0, 0, 0, 0 804 } 805 }, 806 { 807 .addrdec = AMAP_2KB, .dden = DEN_4Gb, .dwid = X8, 808 .rowbits = 16, .colbits = 10, 809 .bits = { 810 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2), 811 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), 812 R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14), 813 R(15), 0, 0, 0 814 } 815 }, 816 { 817 .addrdec = AMAP_2KB, .dden = DEN_8Gb, .dwid = X16, 818 .rowbits = 16, .colbits = 10, 819 .bits = { 820 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2), 821 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), 822 R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14), 823 R(15), 0, 0, 0 824 } 825 }, 826 { 827 .addrdec = AMAP_2KB, .dden = DEN_8Gb, .dwid = X8, 828 .rowbits = 16, .colbits = 11, 829 .bits = { 830 C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2), 831 R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), 832 R(9), R(10), C(8), C(9), R(11), RS, C(11), R(12), R(13), 833 R(14), R(15), 0, 0 834 } 835 }, 836 { 837 .addrdec = AMAP_4KB, .dden = DEN_4Gb, .dwid = X16, 838 .rowbits = 15, .colbits = 10, 839 .bits = { 840 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1), 841 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), 842 R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14), 843 0, 0, 0, 0 844 } 845 }, 846 { 847 .addrdec = AMAP_4KB, .dden = DEN_4Gb, .dwid = X8, 848 .rowbits = 16, .colbits = 10, 849 .bits = { 850 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1), 851 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), 852 R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14), 853 R(15), 0, 0, 0 854 } 855 }, 856 { 857 .addrdec = AMAP_4KB, .dden = DEN_8Gb, .dwid = X16, 858 .rowbits = 16, .colbits = 10, 859 .bits = { 860 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1), 861 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), 862 R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14), 863 R(15), 0, 0, 0 864 } 865 }, 866 { 867 .addrdec = AMAP_4KB, .dden = DEN_8Gb, .dwid = X8, 868 .rowbits = 16, .colbits = 11, 869 .bits = { 870 C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1), 871 B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), 872 R(8), R(9), R(10), C(9), R(11), RS, C(11), R(12), R(13), 873 R(14), R(15), 0, 0 874 } 875 } 876}; 877 878static int bank_hash(u64 pmiaddr, int idx, int shft) 879{ 880 int bhash = 0; 881 882 switch (idx) { 883 case 0: 884 bhash ^= ((pmiaddr >> (12 + shft)) ^ (pmiaddr >> (9 + shft))) & 1; 885 break; 886 case 1: 887 bhash ^= (((pmiaddr >> (10 + shft)) ^ (pmiaddr >> (8 + shft))) & 1) << 1; 888 bhash ^= ((pmiaddr >> 22) & 1) << 1; 889 break; 890 case 2: 891 bhash ^= (((pmiaddr >> (13 + shft)) ^ (pmiaddr >> (11 + shft))) & 1) << 2; 892 break; 893 } 894 895 return bhash; 896} 897 898static int rank_hash(u64 pmiaddr) 899{ 900 return ((pmiaddr >> 16) ^ (pmiaddr >> 10)) & 1; 901} 902 903/* Second stage decode. Compute rank, bank, row & column. */ 904static int apl_pmi2mem(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx, 905 struct dram_addr *daddr, char *msg) 906{ 907 struct d_cr_drp0 *cr_drp0 = &drp0[pmiidx]; 908 struct pnd2_pvt *pvt = mci->pvt_info; 909 int g = pvt->dimm_geom[pmiidx]; 910 struct dimm_geometry *d = &dimms[g]; 911 int column = 0, bank = 0, row = 0, rank = 0; 912 int i, idx, type, skiprs = 0; 913 914 for (i = 0; i < PMI_ADDRESS_WIDTH; i++) { 915 int bit = (pmiaddr >> i) & 1; 916 917 if (i + skiprs >= PMI_ADDRESS_WIDTH) { 918 snprintf(msg, PND2_MSG_SIZE, "Bad dimm_geometry[] table\n"); 919 return -EINVAL; 920 } 921 922 type = d->bits[i + skiprs] & ~0xf; 923 idx = d->bits[i + skiprs] & 0xf; 924 925 /* 926 * On single rank DIMMs ignore the rank select bit 927 * and shift remainder of "bits[]" down one place. 928 */ 929 if (type == RS && (cr_drp0->rken0 + cr_drp0->rken1) == 1) { 930 skiprs = 1; 931 type = d->bits[i + skiprs] & ~0xf; 932 idx = d->bits[i + skiprs] & 0xf; 933 } 934 935 switch (type) { 936 case C(0): 937 column |= (bit << idx); 938 break; 939 case B(0): 940 bank |= (bit << idx); 941 if (cr_drp0->bahen) 942 bank ^= bank_hash(pmiaddr, idx, d->addrdec); 943 break; 944 case R(0): 945 row |= (bit << idx); 946 break; 947 case RS: 948 rank = bit; 949 if (cr_drp0->rsien) 950 rank ^= rank_hash(pmiaddr); 951 break; 952 default: 953 if (bit) { 954 snprintf(msg, PND2_MSG_SIZE, "Bad translation\n"); 955 return -EINVAL; 956 } 957 goto done; 958 } 959 } 960 961done: 962 daddr->col = column; 963 daddr->bank = bank; 964 daddr->row = row; 965 daddr->rank = rank; 966 daddr->dimm = 0; 967 968 return 0; 969} 970 971/* Pluck bit "in" from pmiaddr and return value shifted to bit "out" */ 972#define dnv_get_bit(pmi, in, out) ((int)(((pmi) >> (in)) & 1u) << (out)) 973 974static int dnv_pmi2mem(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx, 975 struct dram_addr *daddr, char *msg) 976{ 977 /* Rank 0 or 1 */ 978 daddr->rank = dnv_get_bit(pmiaddr, dmap[pmiidx].rs0 + 13, 0); 979 /* Rank 2 or 3 */ 980 daddr->rank |= dnv_get_bit(pmiaddr, dmap[pmiidx].rs1 + 13, 1); 981 982 /* 983 * Normally ranks 0,1 are DIMM0, and 2,3 are DIMM1, but we 984 * flip them if DIMM1 is larger than DIMM0. 985 */ 986 daddr->dimm = (daddr->rank >= 2) ^ drp[pmiidx].dimmflip; 987 988 daddr->bank = dnv_get_bit(pmiaddr, dmap[pmiidx].ba0 + 6, 0); 989 daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].ba1 + 6, 1); 990 daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].bg0 + 6, 2); 991 if (dsch.ddr4en) 992 daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].bg1 + 6, 3); 993 if (dmap1[pmiidx].bxor) { 994 if (dsch.ddr4en) { 995 daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 0); 996 daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row7 + 6, 1); 997 if (dsch.chan_width == 0) 998 /* 64/72 bit dram channel width */ 999 daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 2); 1000 else 1001 /* 32/40 bit dram channel width */ 1002 daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 2); 1003 daddr->bank ^= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 3); 1004 } else { 1005 daddr->bank ^= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 0); 1006 daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 1); 1007 if (dsch.chan_width == 0) 1008 daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 2); 1009 else 1010 daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 2); 1011 } 1012 } 1013 1014 daddr->row = dnv_get_bit(pmiaddr, dmap2[pmiidx].row0 + 6, 0); 1015 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row1 + 6, 1); 1016 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 2); 1017 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row3 + 6, 3); 1018 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row4 + 6, 4); 1019 daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row5 + 6, 5); 1020 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 6); 1021 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row7 + 6, 7); 1022 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row8 + 6, 8); 1023 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row9 + 6, 9); 1024 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row10 + 6, 10); 1025 daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row11 + 6, 11); 1026 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row12 + 6, 12); 1027 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row13 + 6, 13); 1028 if (dmap4[pmiidx].row14 != 31) 1029 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row14 + 6, 14); 1030 if (dmap4[pmiidx].row15 != 31) 1031 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row15 + 6, 15); 1032 if (dmap4[pmiidx].row16 != 31) 1033 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row16 + 6, 16); 1034 if (dmap4[pmiidx].row17 != 31) 1035 daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row17 + 6, 17); 1036 1037 daddr->col = dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 3); 1038 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 4); 1039 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca5 + 6, 5); 1040 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca6 + 6, 6); 1041 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca7 + 6, 7); 1042 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca8 + 6, 8); 1043 daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca9 + 6, 9); 1044 if (!dsch.ddr4en && dmap1[pmiidx].ca11 != 0x3f) 1045 daddr->col |= dnv_get_bit(pmiaddr, dmap1[pmiidx].ca11 + 13, 11); 1046 1047 return 0; 1048} 1049 1050static int check_channel(int ch) 1051{ 1052 if (drp0[ch].dramtype != 0) { 1053 pnd2_printk(KERN_INFO, "Unsupported DIMM in channel %d\n", ch); 1054 return 1; 1055 } else if (drp0[ch].eccen == 0) { 1056 pnd2_printk(KERN_INFO, "ECC disabled on channel %d\n", ch); 1057 return 1; 1058 } 1059 return 0; 1060} 1061 1062static int apl_check_ecc_active(void) 1063{ 1064 int i, ret = 0; 1065 1066 /* Check dramtype and ECC mode for each present DIMM */ 1067 for (i = 0; i < APL_NUM_CHANNELS; i++) 1068 if (chan_mask & BIT(i)) 1069 ret += check_channel(i); 1070 return ret ? -EINVAL : 0; 1071} 1072 1073#define DIMMS_PRESENT(d) ((d)->rken0 + (d)->rken1 + (d)->rken2 + (d)->rken3) 1074 1075static int check_unit(int ch) 1076{ 1077 struct d_cr_drp *d = &drp[ch]; 1078 1079 if (DIMMS_PRESENT(d) && !ecc_ctrl[ch].eccen) { 1080 pnd2_printk(KERN_INFO, "ECC disabled on channel %d\n", ch); 1081 return 1; 1082 } 1083 return 0; 1084} 1085 1086static int dnv_check_ecc_active(void) 1087{ 1088 int i, ret = 0; 1089 1090 for (i = 0; i < DNV_NUM_CHANNELS; i++) 1091 ret += check_unit(i); 1092 return ret ? -EINVAL : 0; 1093} 1094 1095static int get_memory_error_data(struct mem_ctl_info *mci, u64 addr, 1096 struct dram_addr *daddr, char *msg) 1097{ 1098 u64 pmiaddr; 1099 u32 pmiidx; 1100 int ret; 1101 1102 ret = sys2pmi(addr, &pmiidx, &pmiaddr, msg); 1103 if (ret) 1104 return ret; 1105 1106 pmiaddr >>= ops->pmiaddr_shift; 1107 /* pmi channel idx to dimm channel idx */ 1108 pmiidx >>= ops->pmiidx_shift; 1109 daddr->chan = pmiidx; 1110 1111 ret = ops->pmi2mem(mci, pmiaddr, pmiidx, daddr, msg); 1112 if (ret) 1113 return ret; 1114 1115 edac_dbg(0, "SysAddr=%llx PmiAddr=%llx Channel=%d DIMM=%d Rank=%d Bank=%d Row=%d Column=%d\n", 1116 addr, pmiaddr, daddr->chan, daddr->dimm, daddr->rank, daddr->bank, daddr->row, daddr->col); 1117 1118 return 0; 1119} 1120 1121static void pnd2_mce_output_error(struct mem_ctl_info *mci, const struct mce *m, 1122 struct dram_addr *daddr) 1123{ 1124 enum hw_event_mc_err_type tp_event; 1125 char *optype, msg[PND2_MSG_SIZE]; 1126 bool ripv = m->mcgstatus & MCG_STATUS_RIPV; 1127 bool overflow = m->status & MCI_STATUS_OVER; 1128 bool uc_err = m->status & MCI_STATUS_UC; 1129 bool recov = m->status & MCI_STATUS_S; 1130 u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52); 1131 u32 mscod = GET_BITFIELD(m->status, 16, 31); 1132 u32 errcode = GET_BITFIELD(m->status, 0, 15); 1133 u32 optypenum = GET_BITFIELD(m->status, 4, 6); 1134 int rc; 1135 1136 tp_event = uc_err ? (ripv ? HW_EVENT_ERR_UNCORRECTED : HW_EVENT_ERR_FATAL) : 1137 HW_EVENT_ERR_CORRECTED; 1138 1139 /* 1140 * According with Table 15-9 of the Intel Architecture spec vol 3A, 1141 * memory errors should fit in this mask: 1142 * 000f 0000 1mmm cccc (binary) 1143 * where: 1144 * f = Correction Report Filtering Bit. If 1, subsequent errors 1145 * won't be shown 1146 * mmm = error type 1147 * cccc = channel 1148 * If the mask doesn't match, report an error to the parsing logic 1149 */ 1150 if (!((errcode & 0xef80) == 0x80)) { 1151 optype = "Can't parse: it is not a mem"; 1152 } else { 1153 switch (optypenum) { 1154 case 0: 1155 optype = "generic undef request error"; 1156 break; 1157 case 1: 1158 optype = "memory read error"; 1159 break; 1160 case 2: 1161 optype = "memory write error"; 1162 break; 1163 case 3: 1164 optype = "addr/cmd error"; 1165 break; 1166 case 4: 1167 optype = "memory scrubbing error"; 1168 break; 1169 default: 1170 optype = "reserved"; 1171 break; 1172 } 1173 } 1174 1175 /* Only decode errors with an valid address (ADDRV) */ 1176 if (!(m->status & MCI_STATUS_ADDRV)) 1177 return; 1178 1179 rc = get_memory_error_data(mci, m->addr, daddr, msg); 1180 if (rc) 1181 goto address_error; 1182 1183 snprintf(msg, sizeof(msg), 1184 "%s%s err_code:%04x:%04x channel:%d DIMM:%d rank:%d row:%d bank:%d col:%d", 1185 overflow ? " OVERFLOW" : "", (uc_err && recov) ? " recoverable" : "", mscod, 1186 errcode, daddr->chan, daddr->dimm, daddr->rank, daddr->row, daddr->bank, daddr->col); 1187 1188 edac_dbg(0, "%s\n", msg); 1189 1190 /* Call the helper to output message */ 1191 edac_mc_handle_error(tp_event, mci, core_err_cnt, m->addr >> PAGE_SHIFT, 1192 m->addr & ~PAGE_MASK, 0, daddr->chan, daddr->dimm, -1, optype, msg); 1193 1194 return; 1195 1196address_error: 1197 edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0, -1, -1, -1, msg, ""); 1198} 1199 1200static void apl_get_dimm_config(struct mem_ctl_info *mci) 1201{ 1202 struct pnd2_pvt *pvt = mci->pvt_info; 1203 struct dimm_info *dimm; 1204 struct d_cr_drp0 *d; 1205 u64 capacity; 1206 int i, g; 1207 1208 for (i = 0; i < APL_NUM_CHANNELS; i++) { 1209 if (!(chan_mask & BIT(i))) 1210 continue; 1211 1212 dimm = edac_get_dimm(mci, i, 0, 0); 1213 if (!dimm) { 1214 edac_dbg(0, "No allocated DIMM for channel %d\n", i); 1215 continue; 1216 } 1217 1218 d = &drp0[i]; 1219 for (g = 0; g < ARRAY_SIZE(dimms); g++) 1220 if (dimms[g].addrdec == d->addrdec && 1221 dimms[g].dden == d->dden && 1222 dimms[g].dwid == d->dwid) 1223 break; 1224 1225 if (g == ARRAY_SIZE(dimms)) { 1226 edac_dbg(0, "Channel %d: unrecognized DIMM\n", i); 1227 continue; 1228 } 1229 1230 pvt->dimm_geom[i] = g; 1231 capacity = (d->rken0 + d->rken1) * 8 * (1ul << dimms[g].rowbits) * 1232 (1ul << dimms[g].colbits); 1233 edac_dbg(0, "Channel %d: %lld MByte DIMM\n", i, capacity >> (20 - 3)); 1234 dimm->nr_pages = MiB_TO_PAGES(capacity >> (20 - 3)); 1235 dimm->grain = 32; 1236 dimm->dtype = (d->dwid == 0) ? DEV_X8 : DEV_X16; 1237 dimm->mtype = MEM_DDR3; 1238 dimm->edac_mode = EDAC_SECDED; 1239 snprintf(dimm->label, sizeof(dimm->label), "Slice#%d_Chan#%d", i / 2, i % 2); 1240 } 1241} 1242 1243static const int dnv_dtypes[] = { 1244 DEV_X8, DEV_X4, DEV_X16, DEV_UNKNOWN 1245}; 1246 1247static void dnv_get_dimm_config(struct mem_ctl_info *mci) 1248{ 1249 int i, j, ranks_of_dimm[DNV_MAX_DIMMS], banks, rowbits, colbits, memtype; 1250 struct dimm_info *dimm; 1251 struct d_cr_drp *d; 1252 u64 capacity; 1253 1254 if (dsch.ddr4en) { 1255 memtype = MEM_DDR4; 1256 banks = 16; 1257 colbits = 10; 1258 } else { 1259 memtype = MEM_DDR3; 1260 banks = 8; 1261 } 1262 1263 for (i = 0; i < DNV_NUM_CHANNELS; i++) { 1264 if (dmap4[i].row14 == 31) 1265 rowbits = 14; 1266 else if (dmap4[i].row15 == 31) 1267 rowbits = 15; 1268 else if (dmap4[i].row16 == 31) 1269 rowbits = 16; 1270 else if (dmap4[i].row17 == 31) 1271 rowbits = 17; 1272 else 1273 rowbits = 18; 1274 1275 if (memtype == MEM_DDR3) { 1276 if (dmap1[i].ca11 != 0x3f) 1277 colbits = 12; 1278 else 1279 colbits = 10; 1280 } 1281 1282 d = &drp[i]; 1283 /* DIMM0 is present if rank0 and/or rank1 is enabled */ 1284 ranks_of_dimm[0] = d->rken0 + d->rken1; 1285 /* DIMM1 is present if rank2 and/or rank3 is enabled */ 1286 ranks_of_dimm[1] = d->rken2 + d->rken3; 1287 1288 for (j = 0; j < DNV_MAX_DIMMS; j++) { 1289 if (!ranks_of_dimm[j]) 1290 continue; 1291 1292 dimm = edac_get_dimm(mci, i, j, 0); 1293 if (!dimm) { 1294 edac_dbg(0, "No allocated DIMM for channel %d DIMM %d\n", i, j); 1295 continue; 1296 } 1297 1298 capacity = ranks_of_dimm[j] * banks * (1ul << rowbits) * (1ul << colbits); 1299 edac_dbg(0, "Channel %d DIMM %d: %lld MByte DIMM\n", i, j, capacity >> (20 - 3)); 1300 dimm->nr_pages = MiB_TO_PAGES(capacity >> (20 - 3)); 1301 dimm->grain = 32; 1302 dimm->dtype = dnv_dtypes[j ? d->dimmdwid0 : d->dimmdwid1]; 1303 dimm->mtype = memtype; 1304 dimm->edac_mode = EDAC_SECDED; 1305 snprintf(dimm->label, sizeof(dimm->label), "Chan#%d_DIMM#%d", i, j); 1306 } 1307 } 1308} 1309 1310static int pnd2_register_mci(struct mem_ctl_info **ppmci) 1311{ 1312 struct edac_mc_layer layers[2]; 1313 struct mem_ctl_info *mci; 1314 struct pnd2_pvt *pvt; 1315 int rc; 1316 1317 rc = ops->check_ecc(); 1318 if (rc < 0) 1319 return rc; 1320 1321 /* Allocate a new MC control structure */ 1322 layers[0].type = EDAC_MC_LAYER_CHANNEL; 1323 layers[0].size = ops->channels; 1324 layers[0].is_virt_csrow = false; 1325 layers[1].type = EDAC_MC_LAYER_SLOT; 1326 layers[1].size = ops->dimms_per_channel; 1327 layers[1].is_virt_csrow = true; 1328 mci = edac_mc_alloc(0, ARRAY_SIZE(layers), layers, sizeof(*pvt)); 1329 if (!mci) 1330 return -ENOMEM; 1331 1332 pvt = mci->pvt_info; 1333 memset(pvt, 0, sizeof(*pvt)); 1334 1335 mci->mod_name = EDAC_MOD_STR; 1336 mci->dev_name = ops->name; 1337 mci->ctl_name = "Pondicherry2"; 1338 1339 /* Get dimm basic config and the memory layout */ 1340 ops->get_dimm_config(mci); 1341 1342 if (edac_mc_add_mc(mci)) { 1343 edac_dbg(0, "MC: failed edac_mc_add_mc()\n"); 1344 edac_mc_free(mci); 1345 return -EINVAL; 1346 } 1347 1348 *ppmci = mci; 1349 1350 return 0; 1351} 1352 1353static void pnd2_unregister_mci(struct mem_ctl_info *mci) 1354{ 1355 if (unlikely(!mci || !mci->pvt_info)) { 1356 pnd2_printk(KERN_ERR, "Couldn't find mci handler\n"); 1357 return; 1358 } 1359 1360 /* Remove MC sysfs nodes */ 1361 edac_mc_del_mc(NULL); 1362 edac_dbg(1, "%s: free mci struct\n", mci->ctl_name); 1363 edac_mc_free(mci); 1364} 1365 1366/* 1367 * Callback function registered with core kernel mce code. 1368 * Called once for each logged error. 1369 */ 1370static int pnd2_mce_check_error(struct notifier_block *nb, unsigned long val, void *data) 1371{ 1372 struct mce *mce = (struct mce *)data; 1373 struct mem_ctl_info *mci; 1374 struct dram_addr daddr; 1375 char *type; 1376 1377 mci = pnd2_mci; 1378 if (!mci || (mce->kflags & MCE_HANDLED_CEC)) 1379 return NOTIFY_DONE; 1380 1381 /* 1382 * Just let mcelog handle it if the error is 1383 * outside the memory controller. A memory error 1384 * is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0. 1385 * bit 12 has an special meaning. 1386 */ 1387 if ((mce->status & 0xefff) >> 7 != 1) 1388 return NOTIFY_DONE; 1389 1390 if (mce->mcgstatus & MCG_STATUS_MCIP) 1391 type = "Exception"; 1392 else 1393 type = "Event"; 1394 1395 pnd2_mc_printk(mci, KERN_INFO, "HANDLING MCE MEMORY ERROR\n"); 1396 pnd2_mc_printk(mci, KERN_INFO, "CPU %u: Machine Check %s: %llx Bank %u: %llx\n", 1397 mce->extcpu, type, mce->mcgstatus, mce->bank, mce->status); 1398 pnd2_mc_printk(mci, KERN_INFO, "TSC %llx ", mce->tsc); 1399 pnd2_mc_printk(mci, KERN_INFO, "ADDR %llx ", mce->addr); 1400 pnd2_mc_printk(mci, KERN_INFO, "MISC %llx ", mce->misc); 1401 pnd2_mc_printk(mci, KERN_INFO, "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n", 1402 mce->cpuvendor, mce->cpuid, mce->time, mce->socketid, mce->apicid); 1403 1404 pnd2_mce_output_error(mci, mce, &daddr); 1405 1406 /* Advice mcelog that the error were handled */ 1407 mce->kflags |= MCE_HANDLED_EDAC; 1408 return NOTIFY_OK; 1409} 1410 1411static struct notifier_block pnd2_mce_dec = { 1412 .notifier_call = pnd2_mce_check_error, 1413 .priority = MCE_PRIO_EDAC, 1414}; 1415 1416#ifdef CONFIG_EDAC_DEBUG 1417/* 1418 * Write an address to this file to exercise the address decode 1419 * logic in this driver. 1420 */ 1421static u64 pnd2_fake_addr; 1422#define PND2_BLOB_SIZE 1024 1423static char pnd2_result[PND2_BLOB_SIZE]; 1424static struct dentry *pnd2_test; 1425static struct debugfs_blob_wrapper pnd2_blob = { 1426 .data = pnd2_result, 1427 .size = 0 1428}; 1429 1430static int debugfs_u64_set(void *data, u64 val) 1431{ 1432 struct dram_addr daddr; 1433 struct mce m; 1434 1435 *(u64 *)data = val; 1436 m.mcgstatus = 0; 1437 /* ADDRV + MemRd + Unknown channel */ 1438 m.status = MCI_STATUS_ADDRV + 0x9f; 1439 m.addr = val; 1440 pnd2_mce_output_error(pnd2_mci, &m, &daddr); 1441 snprintf(pnd2_blob.data, PND2_BLOB_SIZE, 1442 "SysAddr=%llx Channel=%d DIMM=%d Rank=%d Bank=%d Row=%d Column=%d\n", 1443 m.addr, daddr.chan, daddr.dimm, daddr.rank, daddr.bank, daddr.row, daddr.col); 1444 pnd2_blob.size = strlen(pnd2_blob.data); 1445 1446 return 0; 1447} 1448DEFINE_DEBUGFS_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); 1449 1450static void setup_pnd2_debug(void) 1451{ 1452 pnd2_test = edac_debugfs_create_dir("pnd2_test"); 1453 edac_debugfs_create_file("pnd2_debug_addr", 0200, pnd2_test, 1454 &pnd2_fake_addr, &fops_u64_wo); 1455 debugfs_create_blob("pnd2_debug_results", 0400, pnd2_test, &pnd2_blob); 1456} 1457 1458static void teardown_pnd2_debug(void) 1459{ 1460 debugfs_remove_recursive(pnd2_test); 1461} 1462#else 1463static void setup_pnd2_debug(void) {} 1464static void teardown_pnd2_debug(void) {} 1465#endif /* CONFIG_EDAC_DEBUG */ 1466 1467 1468static int pnd2_probe(void) 1469{ 1470 int rc; 1471 1472 edac_dbg(2, "\n"); 1473 rc = get_registers(); 1474 if (rc) 1475 return rc; 1476 1477 return pnd2_register_mci(&pnd2_mci); 1478} 1479 1480static void pnd2_remove(void) 1481{ 1482 edac_dbg(0, "\n"); 1483 pnd2_unregister_mci(pnd2_mci); 1484} 1485 1486static struct dunit_ops apl_ops = { 1487 .name = "pnd2/apl", 1488 .type = APL, 1489 .pmiaddr_shift = LOG2_PMI_ADDR_GRANULARITY, 1490 .pmiidx_shift = 0, 1491 .channels = APL_NUM_CHANNELS, 1492 .dimms_per_channel = 1, 1493 .rd_reg = apl_rd_reg, 1494 .get_registers = apl_get_registers, 1495 .check_ecc = apl_check_ecc_active, 1496 .mk_region = apl_mk_region, 1497 .get_dimm_config = apl_get_dimm_config, 1498 .pmi2mem = apl_pmi2mem, 1499}; 1500 1501static struct dunit_ops dnv_ops = { 1502 .name = "pnd2/dnv", 1503 .type = DNV, 1504 .pmiaddr_shift = 0, 1505 .pmiidx_shift = 1, 1506 .channels = DNV_NUM_CHANNELS, 1507 .dimms_per_channel = 2, 1508 .rd_reg = dnv_rd_reg, 1509 .get_registers = dnv_get_registers, 1510 .check_ecc = dnv_check_ecc_active, 1511 .mk_region = dnv_mk_region, 1512 .get_dimm_config = dnv_get_dimm_config, 1513 .pmi2mem = dnv_pmi2mem, 1514}; 1515 1516static const struct x86_cpu_id pnd2_cpuids[] = { 1517 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &apl_ops), 1518 X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &dnv_ops), 1519 { } 1520}; 1521MODULE_DEVICE_TABLE(x86cpu, pnd2_cpuids); 1522 1523static int __init pnd2_init(void) 1524{ 1525 const struct x86_cpu_id *id; 1526 const char *owner; 1527 int rc; 1528 1529 edac_dbg(2, "\n"); 1530 1531 if (ghes_get_devices()) 1532 return -EBUSY; 1533 1534 owner = edac_get_owner(); 1535 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) 1536 return -EBUSY; 1537 1538 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) 1539 return -ENODEV; 1540 1541 id = x86_match_cpu(pnd2_cpuids); 1542 if (!id) 1543 return -ENODEV; 1544 1545 ops = (struct dunit_ops *)id->driver_data; 1546 1547 if (ops->type == APL) { 1548 p2sb_bus = pci_find_bus(0, 0); 1549 if (!p2sb_bus) 1550 return -ENODEV; 1551 } 1552 1553 /* Ensure that the OPSTATE is set correctly for POLL or NMI */ 1554 opstate_init(); 1555 1556 rc = pnd2_probe(); 1557 if (rc < 0) { 1558 pnd2_printk(KERN_ERR, "Failed to register device with error %d.\n", rc); 1559 return rc; 1560 } 1561 1562 if (!pnd2_mci) 1563 return -ENODEV; 1564 1565 mce_register_decode_chain(&pnd2_mce_dec); 1566 setup_pnd2_debug(); 1567 1568 return 0; 1569} 1570 1571static void __exit pnd2_exit(void) 1572{ 1573 edac_dbg(2, "\n"); 1574 teardown_pnd2_debug(); 1575 mce_unregister_decode_chain(&pnd2_mce_dec); 1576 pnd2_remove(); 1577} 1578 1579module_init(pnd2_init); 1580module_exit(pnd2_exit); 1581 1582module_param(edac_op_state, int, 0444); 1583MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI"); 1584 1585MODULE_LICENSE("GPL v2"); 1586MODULE_AUTHOR("Tony Luck"); 1587MODULE_DESCRIPTION("MC Driver for Intel SoC using Pondicherry memory controller"); 1588