drivers/acpi/nfit.c

   1 /*
   2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #include <linux/list_sort.h>
  14 #include <linux/libnvdimm.h>
  15 #include <linux/module.h>
  16 #include <linux/mutex.h>
  17 #include <linux/ndctl.h>
  18 #include <linux/list.h>
  19 #include <linux/acpi.h>
  20 #include <linux/sort.h>
  21 #include <linux/io.h>
  22 #include "nfit.h"
  23
  24 /*
  25  * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
  26  * irrelevant.
  27  */
  28 #include <asm-generic/io-64-nonatomic-hi-lo.h>
  29
  30 static bool force_enable_dimms;
  31 module_param(force_enable_dimms, bool, S_IRUGO|S_IWUSR);
  32 MODULE_PARM_DESC(force_enable_dimms, "Ignore _STA (ACPI DIMM device) status");
  33
  34 static u8 nfit_uuid[NFIT_UUID_MAX][16];
  35
  36 const u8 *to_nfit_uuid(enum nfit_uuids id)
  37 {
  38         return nfit_uuid[id];
  39 }
  40 EXPORT_SYMBOL(to_nfit_uuid);
  41
  42 static struct acpi_nfit_desc *to_acpi_nfit_desc(
  43                 struct nvdimm_bus_descriptor *nd_desc)
  44 {
  45         return container_of(nd_desc, struct acpi_nfit_desc, nd_desc);
  46 }
  47
  48 static struct acpi_device *to_acpi_dev(struct acpi_nfit_desc *acpi_desc)
  49 {
  50         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
  51
  52         /*
  53          * If provider == 'ACPI.NFIT' we can assume 'dev' is a struct
  54          * acpi_device.
  55          */
  56         if (!nd_desc->provider_name
  57                         || strcmp(nd_desc->provider_name, "ACPI.NFIT") != 0)
  58                 return NULL;
  59
  60         return to_acpi_device(acpi_desc->dev);
  61 }
  62
  63 static int acpi_nfit_ctl(struct nvdimm_bus_descriptor *nd_desc,
  64                 struct nvdimm *nvdimm, unsigned int cmd, void *buf,
  65                 unsigned int buf_len)
  66 {
  67         struct acpi_nfit_desc *acpi_desc = to_acpi_nfit_desc(nd_desc);
  68         const struct nd_cmd_desc *desc = NULL;
  69         union acpi_object in_obj, in_buf, *out_obj;
  70         struct device *dev = acpi_desc->dev;
  71         const char *cmd_name, *dimm_name;
  72         unsigned long dsm_mask;
  73         acpi_handle handle;
  74         const u8 *uuid;
  75         u32 offset;
  76         int rc, i;
  77
  78         if (nvdimm) {
  79                 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
  80                 struct acpi_device *adev = nfit_mem->adev;
  81
  82                 if (!adev)
  83                         return -ENOTTY;
  84                 dimm_name = nvdimm_name(nvdimm);
  85                 cmd_name = nvdimm_cmd_name(cmd);
  86                 dsm_mask = nfit_mem->dsm_mask;
  87                 desc = nd_cmd_dimm_desc(cmd);
  88                 uuid = to_nfit_uuid(NFIT_DEV_DIMM);
  89                 handle = adev->handle;
  90         } else {
  91                 struct acpi_device *adev = to_acpi_dev(acpi_desc);
  92
  93                 cmd_name = nvdimm_bus_cmd_name(cmd);
  94                 dsm_mask = nd_desc->dsm_mask;
  95                 desc = nd_cmd_bus_desc(cmd);
  96                 uuid = to_nfit_uuid(NFIT_DEV_BUS);
  97                 handle = adev->handle;
  98                 dimm_name = "bus";
  99         }
 100
 101         if (!desc || (cmd && (desc->out_num + desc->in_num == 0)))
 102                 return -ENOTTY;
 103
 104         if (!test_bit(cmd, &dsm_mask))
 105                 return -ENOTTY;
 106
 107         in_obj.type = ACPI_TYPE_PACKAGE;
 108         in_obj.package.count = 1;
 109         in_obj.package.elements = &in_buf;
 110         in_buf.type = ACPI_TYPE_BUFFER;
 111         in_buf.buffer.pointer = buf;
 112         in_buf.buffer.length = 0;
 113
 114         /* libnvdimm has already validated the input envelope */
 115         for (i = 0; i < desc->in_num; i++)
 116                 in_buf.buffer.length += nd_cmd_in_size(nvdimm, cmd, desc,
 117                                 i, buf);
 118
 119         if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
 120                 dev_dbg(dev, "%s:%s cmd: %s input length: %d\n", __func__,
 121                                 dimm_name, cmd_name, in_buf.buffer.length);
 122                 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
 123                                 4, in_buf.buffer.pointer, min_t(u32, 128,
 124                                         in_buf.buffer.length), true);
 125         }
 126
 127         out_obj = acpi_evaluate_dsm(handle, uuid, 1, cmd, &in_obj);
 128         if (!out_obj) {
 129                 dev_dbg(dev, "%s:%s _DSM failed cmd: %s\n", __func__, dimm_name,
 130                                 cmd_name);
 131                 return -EINVAL;
 132         }
 133
 134         if (out_obj->package.type != ACPI_TYPE_BUFFER) {
 135                 dev_dbg(dev, "%s:%s unexpected output object type cmd: %s type: %d\n",
 136                                 __func__, dimm_name, cmd_name, out_obj->type);
 137                 rc = -EINVAL;
 138                 goto out;
 139         }
 140
 141         if (IS_ENABLED(CONFIG_ACPI_NFIT_DEBUG)) {
 142                 dev_dbg(dev, "%s:%s cmd: %s output length: %d\n", __func__,
 143                                 dimm_name, cmd_name, out_obj->buffer.length);
 144                 print_hex_dump_debug(cmd_name, DUMP_PREFIX_OFFSET, 4,
 145                                 4, out_obj->buffer.pointer, min_t(u32, 128,
 146                                         out_obj->buffer.length), true);
 147         }
 148
 149         for (i = 0, offset = 0; i < desc->out_num; i++) {
 150                 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i, buf,
 151                                 (u32 *) out_obj->buffer.pointer);
 152
 153                 if (offset + out_size > out_obj->buffer.length) {
 154                         dev_dbg(dev, "%s:%s output object underflow cmd: %s field: %d\n",
 155                                         __func__, dimm_name, cmd_name, i);
 156                         break;
 157                 }
 158
 159                 if (in_buf.buffer.length + offset + out_size > buf_len) {
 160                         dev_dbg(dev, "%s:%s output overrun cmd: %s field: %d\n",
 161                                         __func__, dimm_name, cmd_name, i);
 162                         rc = -ENXIO;
 163                         goto out;
 164                 }
 165                 memcpy(buf + in_buf.buffer.length + offset,
 166                                 out_obj->buffer.pointer + offset, out_size);
 167                 offset += out_size;
 168         }
 169         if (offset + in_buf.buffer.length < buf_len) {
 170                 if (i >= 1) {
 171                         /*
 172                          * status valid, return the number of bytes left
 173                          * unfilled in the output buffer
 174                          */
 175                         rc = buf_len - offset - in_buf.buffer.length;
 176                 } else {
 177                         dev_err(dev, "%s:%s underrun cmd: %s buf_len: %d out_len: %d\n",
 178                                         __func__, dimm_name, cmd_name, buf_len,
 179                                         offset);
 180                         rc = -ENXIO;
 181                 }
 182         } else
 183                 rc = 0;
 184
 185  out:
 186         ACPI_FREE(out_obj);
 187
 188         return rc;
 189 }
 190
 191 static const char *spa_type_name(u16 type)
 192 {
 193         static const char *to_name[] = {
 194                 [NFIT_SPA_VOLATILE] = "volatile",
 195                 [NFIT_SPA_PM] = "pmem",
 196                 [NFIT_SPA_DCR] = "dimm-control-region",
 197                 [NFIT_SPA_BDW] = "block-data-window",
 198                 [NFIT_SPA_VDISK] = "volatile-disk",
 199                 [NFIT_SPA_VCD] = "volatile-cd",
 200                 [NFIT_SPA_PDISK] = "persistent-disk",
 201                 [NFIT_SPA_PCD] = "persistent-cd",
 202
 203         };
 204
 205         if (type > NFIT_SPA_PCD)
 206                 return "unknown";
 207
 208         return to_name[type];
 209 }
 210
 211 static int nfit_spa_type(struct acpi_nfit_system_address *spa)
 212 {
 213         int i;
 214
 215         for (i = 0; i < NFIT_UUID_MAX; i++)
 216                 if (memcmp(to_nfit_uuid(i), spa->range_guid, 16) == 0)
 217                         return i;
 218         return -1;
 219 }
 220
 221 static bool add_spa(struct acpi_nfit_desc *acpi_desc,
 222                 struct acpi_nfit_system_address *spa)
 223 {
 224         struct device *dev = acpi_desc->dev;
 225         struct nfit_spa *nfit_spa = devm_kzalloc(dev, sizeof(*nfit_spa),
 226                         GFP_KERNEL);
 227
 228         if (!nfit_spa)
 229                 return false;
 230         INIT_LIST_HEAD(&nfit_spa->list);
 231         nfit_spa->spa = spa;
 232         list_add_tail(&nfit_spa->list, &acpi_desc->spas);
 233         dev_dbg(dev, "%s: spa index: %d type: %s\n", __func__,
 234                         spa->range_index,
 235                         spa_type_name(nfit_spa_type(spa)));
 236         return true;
 237 }
 238
 239 static bool add_memdev(struct acpi_nfit_desc *acpi_desc,
 240                 struct acpi_nfit_memory_map *memdev)
 241 {
 242         struct device *dev = acpi_desc->dev;
 243         struct nfit_memdev *nfit_memdev = devm_kzalloc(dev,
 244                         sizeof(*nfit_memdev), GFP_KERNEL);
 245
 246         if (!nfit_memdev)
 247                 return false;
 248         INIT_LIST_HEAD(&nfit_memdev->list);
 249         nfit_memdev->memdev = memdev;
 250         list_add_tail(&nfit_memdev->list, &acpi_desc->memdevs);
 251         dev_dbg(dev, "%s: memdev handle: %#x spa: %d dcr: %d\n",
 252                         __func__, memdev->device_handle, memdev->range_index,
 253                         memdev->region_index);
 254         return true;
 255 }
 256
 257 static bool add_dcr(struct acpi_nfit_desc *acpi_desc,
 258                 struct acpi_nfit_control_region *dcr)
 259 {
 260         struct device *dev = acpi_desc->dev;
 261         struct nfit_dcr *nfit_dcr = devm_kzalloc(dev, sizeof(*nfit_dcr),
 262                         GFP_KERNEL);
 263
 264         if (!nfit_dcr)
 265                 return false;
 266         INIT_LIST_HEAD(&nfit_dcr->list);
 267         nfit_dcr->dcr = dcr;
 268         list_add_tail(&nfit_dcr->list, &acpi_desc->dcrs);
 269         dev_dbg(dev, "%s: dcr index: %d windows: %d\n", __func__,
 270                         dcr->region_index, dcr->windows);
 271         return true;
 272 }
 273
 274 static bool add_bdw(struct acpi_nfit_desc *acpi_desc,
 275                 struct acpi_nfit_data_region *bdw)
 276 {
 277         struct device *dev = acpi_desc->dev;
 278         struct nfit_bdw *nfit_bdw = devm_kzalloc(dev, sizeof(*nfit_bdw),
 279                         GFP_KERNEL);
 280
 281         if (!nfit_bdw)
 282                 return false;
 283         INIT_LIST_HEAD(&nfit_bdw->list);
 284         nfit_bdw->bdw = bdw;
 285         list_add_tail(&nfit_bdw->list, &acpi_desc->bdws);
 286         dev_dbg(dev, "%s: bdw dcr: %d windows: %d\n", __func__,
 287                         bdw->region_index, bdw->windows);
 288         return true;
 289 }
 290
 291 static bool add_idt(struct acpi_nfit_desc *acpi_desc,
 292                 struct acpi_nfit_interleave *idt)
 293 {
 294         struct device *dev = acpi_desc->dev;
 295         struct nfit_idt *nfit_idt = devm_kzalloc(dev, sizeof(*nfit_idt),
 296                         GFP_KERNEL);
 297
 298         if (!nfit_idt)
 299                 return false;
 300         INIT_LIST_HEAD(&nfit_idt->list);
 301         nfit_idt->idt = idt;
 302         list_add_tail(&nfit_idt->list, &acpi_desc->idts);
 303         dev_dbg(dev, "%s: idt index: %d num_lines: %d\n", __func__,
 304                         idt->interleave_index, idt->line_count);
 305         return true;
 306 }
 307
 308 static void *add_table(struct acpi_nfit_desc *acpi_desc, void *table,
 309                 const void *end)
 310 {
 311         struct device *dev = acpi_desc->dev;
 312         struct acpi_nfit_header *hdr;
 313         void *err = ERR_PTR(-ENOMEM);
 314
 315         if (table >= end)
 316                 return NULL;
 317
 318         hdr = table;
 319         switch (hdr->type) {
 320         case ACPI_NFIT_TYPE_SYSTEM_ADDRESS:
 321                 if (!add_spa(acpi_desc, table))
 322                         return err;
 323                 break;
 324         case ACPI_NFIT_TYPE_MEMORY_MAP:
 325                 if (!add_memdev(acpi_desc, table))
 326                         return err;
 327                 break;
 328         case ACPI_NFIT_TYPE_CONTROL_REGION:
 329                 if (!add_dcr(acpi_desc, table))
 330                         return err;
 331                 break;
 332         case ACPI_NFIT_TYPE_DATA_REGION:
 333                 if (!add_bdw(acpi_desc, table))
 334                         return err;
 335                 break;
 336         case ACPI_NFIT_TYPE_INTERLEAVE:
 337                 if (!add_idt(acpi_desc, table))
 338                         return err;
 339                 break;
 340         case ACPI_NFIT_TYPE_FLUSH_ADDRESS:
 341                 dev_dbg(dev, "%s: flush\n", __func__);
 342                 break;
 343         case ACPI_NFIT_TYPE_SMBIOS:
 344                 dev_dbg(dev, "%s: smbios\n", __func__);
 345                 break;
 346         default:
 347                 dev_err(dev, "unknown table '%d' parsing nfit\n", hdr->type);
 348                 break;
 349         }
 350
 351         return table + hdr->length;
 352 }
 353
 354 static void nfit_mem_find_spa_bdw(struct acpi_nfit_desc *acpi_desc,
 355                 struct nfit_mem *nfit_mem)
 356 {
 357         u32 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 358         u16 dcr = nfit_mem->dcr->region_index;
 359         struct nfit_spa *nfit_spa;
 360
 361         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 362                 u16 range_index = nfit_spa->spa->range_index;
 363                 int type = nfit_spa_type(nfit_spa->spa);
 364                 struct nfit_memdev *nfit_memdev;
 365
 366                 if (type != NFIT_SPA_BDW)
 367                         continue;
 368
 369                 list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 370                         if (nfit_memdev->memdev->range_index != range_index)
 371                                 continue;
 372                         if (nfit_memdev->memdev->device_handle != device_handle)
 373                                 continue;
 374                         if (nfit_memdev->memdev->region_index != dcr)
 375                                 continue;
 376
 377                         nfit_mem->spa_bdw = nfit_spa->spa;
 378                         return;
 379                 }
 380         }
 381
 382         dev_dbg(acpi_desc->dev, "SPA-BDW not found for SPA-DCR %d\n",
 383                         nfit_mem->spa_dcr->range_index);
 384         nfit_mem->bdw = NULL;
 385 }
 386
 387 static int nfit_mem_add(struct acpi_nfit_desc *acpi_desc,
 388                 struct nfit_mem *nfit_mem, struct acpi_nfit_system_address *spa)
 389 {
 390         u16 dcr = __to_nfit_memdev(nfit_mem)->region_index;
 391         struct nfit_memdev *nfit_memdev;
 392         struct nfit_dcr *nfit_dcr;
 393         struct nfit_bdw *nfit_bdw;
 394         struct nfit_idt *nfit_idt;
 395         u16 idt_idx, range_index;
 396
 397         list_for_each_entry(nfit_dcr, &acpi_desc->dcrs, list) {
 398                 if (nfit_dcr->dcr->region_index != dcr)
 399                         continue;
 400                 nfit_mem->dcr = nfit_dcr->dcr;
 401                 break;
 402         }
 403
 404         if (!nfit_mem->dcr) {
 405                 dev_dbg(acpi_desc->dev, "SPA %d missing:%s%s\n",
 406                                 spa->range_index, __to_nfit_memdev(nfit_mem)
 407                                 ? "" : " MEMDEV", nfit_mem->dcr ? "" : " DCR");
 408                 return -ENODEV;
 409         }
 410
 411         /*
 412          * We've found enough to create an nvdimm, optionally
 413          * find an associated BDW
 414          */
 415         list_add(&nfit_mem->list, &acpi_desc->dimms);
 416
 417         list_for_each_entry(nfit_bdw, &acpi_desc->bdws, list) {
 418                 if (nfit_bdw->bdw->region_index != dcr)
 419                         continue;
 420                 nfit_mem->bdw = nfit_bdw->bdw;
 421                 break;
 422         }
 423
 424         if (!nfit_mem->bdw)
 425                 return 0;
 426
 427         nfit_mem_find_spa_bdw(acpi_desc, nfit_mem);
 428
 429         if (!nfit_mem->spa_bdw)
 430                 return 0;
 431
 432         range_index = nfit_mem->spa_bdw->range_index;
 433         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 434                 if (nfit_memdev->memdev->range_index != range_index ||
 435                                 nfit_memdev->memdev->region_index != dcr)
 436                         continue;
 437                 nfit_mem->memdev_bdw = nfit_memdev->memdev;
 438                 idt_idx = nfit_memdev->memdev->interleave_index;
 439                 list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
 440                         if (nfit_idt->idt->interleave_index != idt_idx)
 441                                 continue;
 442                         nfit_mem->idt_bdw = nfit_idt->idt;
 443                         break;
 444                 }
 445                 break;
 446         }
 447
 448         return 0;
 449 }
 450
 451 static int nfit_mem_dcr_init(struct acpi_nfit_desc *acpi_desc,
 452                 struct acpi_nfit_system_address *spa)
 453 {
 454         struct nfit_mem *nfit_mem, *found;
 455         struct nfit_memdev *nfit_memdev;
 456         int type = nfit_spa_type(spa);
 457         u16 dcr;
 458
 459         switch (type) {
 460         case NFIT_SPA_DCR:
 461         case NFIT_SPA_PM:
 462                 break;
 463         default:
 464                 return 0;
 465         }
 466
 467         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
 468                 int rc;
 469
 470                 if (nfit_memdev->memdev->range_index != spa->range_index)
 471                         continue;
 472                 found = NULL;
 473                 dcr = nfit_memdev->memdev->region_index;
 474                 list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
 475                         if (__to_nfit_memdev(nfit_mem)->region_index == dcr) {
 476                                 found = nfit_mem;
 477                                 break;
 478                         }
 479
 480                 if (found)
 481                         nfit_mem = found;
 482                 else {
 483                         nfit_mem = devm_kzalloc(acpi_desc->dev,
 484                                         sizeof(*nfit_mem), GFP_KERNEL);
 485                         if (!nfit_mem)
 486                                 return -ENOMEM;
 487                         INIT_LIST_HEAD(&nfit_mem->list);
 488                 }
 489
 490                 if (type == NFIT_SPA_DCR) {
 491                         struct nfit_idt *nfit_idt;
 492                         u16 idt_idx;
 493
 494                         /* multiple dimms may share a SPA when interleaved */
 495                         nfit_mem->spa_dcr = spa;
 496                         nfit_mem->memdev_dcr = nfit_memdev->memdev;
 497                         idt_idx = nfit_memdev->memdev->interleave_index;
 498                         list_for_each_entry(nfit_idt, &acpi_desc->idts, list) {
 499                                 if (nfit_idt->idt->interleave_index != idt_idx)
 500                                         continue;
 501                                 nfit_mem->idt_dcr = nfit_idt->idt;
 502                                 break;
 503                         }
 504                 } else {
 505                         /*
 506                          * A single dimm may belong to multiple SPA-PM
 507                          * ranges, record at least one in addition to
 508                          * any SPA-DCR range.
 509                          */
 510                         nfit_mem->memdev_pmem = nfit_memdev->memdev;
 511                 }
 512
 513                 if (found)
 514                         continue;
 515
 516                 rc = nfit_mem_add(acpi_desc, nfit_mem, spa);
 517                 if (rc)
 518                         return rc;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static int nfit_mem_cmp(void *priv, struct list_head *_a, struct list_head *_b)
 525 {
 526         struct nfit_mem *a = container_of(_a, typeof(*a), list);
 527         struct nfit_mem *b = container_of(_b, typeof(*b), list);
 528         u32 handleA, handleB;
 529
 530         handleA = __to_nfit_memdev(a)->device_handle;
 531         handleB = __to_nfit_memdev(b)->device_handle;
 532         if (handleA < handleB)
 533                 return -1;
 534         else if (handleA > handleB)
 535                 return 1;
 536         return 0;
 537 }
 538
 539 static int nfit_mem_init(struct acpi_nfit_desc *acpi_desc)
 540 {
 541         struct nfit_spa *nfit_spa;
 542
 543         /*
 544          * For each SPA-DCR or SPA-PMEM address range find its
 545          * corresponding MEMDEV(s).  From each MEMDEV find the
 546          * corresponding DCR.  Then, if we're operating on a SPA-DCR,
 547          * try to find a SPA-BDW and a corresponding BDW that references
 548          * the DCR.  Throw it all into an nfit_mem object.  Note, that
 549          * BDWs are optional.
 550          */
 551         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
 552                 int rc;
 553
 554                 rc = nfit_mem_dcr_init(acpi_desc, nfit_spa->spa);
 555                 if (rc)
 556                         return rc;
 557         }
 558
 559         list_sort(NULL, &acpi_desc->dimms, nfit_mem_cmp);
 560
 561         return 0;
 562 }
 563
 564 static ssize_t revision_show(struct device *dev,
 565                 struct device_attribute *attr, char *buf)
 566 {
 567         struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
 568         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
 569         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
 570
 571         return sprintf(buf, "%d\n", acpi_desc->nfit->header.revision);
 572 }
 573 static DEVICE_ATTR_RO(revision);
 574
 575 static struct attribute *acpi_nfit_attributes[] = {
 576         &dev_attr_revision.attr,
 577         NULL,
 578 };
 579
 580 static struct attribute_group acpi_nfit_attribute_group = {
 581         .name = "nfit",
 582         .attrs = acpi_nfit_attributes,
 583 };
 584
 585 const struct attribute_group *acpi_nfit_attribute_groups[] = {
 586         &nvdimm_bus_attribute_group,
 587         &acpi_nfit_attribute_group,
 588         NULL,
 589 };
 590 EXPORT_SYMBOL_GPL(acpi_nfit_attribute_groups);
 591
 592 static struct acpi_nfit_memory_map *to_nfit_memdev(struct device *dev)
 593 {
 594         struct nvdimm *nvdimm = to_nvdimm(dev);
 595         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 596
 597         return __to_nfit_memdev(nfit_mem);
 598 }
 599
 600 static struct acpi_nfit_control_region *to_nfit_dcr(struct device *dev)
 601 {
 602         struct nvdimm *nvdimm = to_nvdimm(dev);
 603         struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 604
 605         return nfit_mem->dcr;
 606 }
 607
 608 static ssize_t handle_show(struct device *dev,
 609                 struct device_attribute *attr, char *buf)
 610 {
 611         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
 612
 613         return sprintf(buf, "%#x\n", memdev->device_handle);
 614 }
 615 static DEVICE_ATTR_RO(handle);
 616
 617 static ssize_t phys_id_show(struct device *dev,
 618                 struct device_attribute *attr, char *buf)
 619 {
 620         struct acpi_nfit_memory_map *memdev = to_nfit_memdev(dev);
 621
 622         return sprintf(buf, "%#x\n", memdev->physical_id);
 623 }
 624 static DEVICE_ATTR_RO(phys_id);
 625
 626 static ssize_t vendor_show(struct device *dev,
 627                 struct device_attribute *attr, char *buf)
 628 {
 629         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 630
 631         return sprintf(buf, "%#x\n", dcr->vendor_id);
 632 }
 633 static DEVICE_ATTR_RO(vendor);
 634
 635 static ssize_t rev_id_show(struct device *dev,
 636                 struct device_attribute *attr, char *buf)
 637 {
 638         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 639
 640         return sprintf(buf, "%#x\n", dcr->revision_id);
 641 }
 642 static DEVICE_ATTR_RO(rev_id);
 643
 644 static ssize_t device_show(struct device *dev,
 645                 struct device_attribute *attr, char *buf)
 646 {
 647         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 648
 649         return sprintf(buf, "%#x\n", dcr->device_id);
 650 }
 651 static DEVICE_ATTR_RO(device);
 652
 653 static ssize_t format_show(struct device *dev,
 654                 struct device_attribute *attr, char *buf)
 655 {
 656         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 657
 658         return sprintf(buf, "%#x\n", dcr->code);
 659 }
 660 static DEVICE_ATTR_RO(format);
 661
 662 static ssize_t serial_show(struct device *dev,
 663                 struct device_attribute *attr, char *buf)
 664 {
 665         struct acpi_nfit_control_region *dcr = to_nfit_dcr(dev);
 666
 667         return sprintf(buf, "%#x\n", dcr->serial_number);
 668 }
 669 static DEVICE_ATTR_RO(serial);
 670
 671 static struct attribute *acpi_nfit_dimm_attributes[] = {
 672         &dev_attr_handle.attr,
 673         &dev_attr_phys_id.attr,
 674         &dev_attr_vendor.attr,
 675         &dev_attr_device.attr,
 676         &dev_attr_format.attr,
 677         &dev_attr_serial.attr,
 678         &dev_attr_rev_id.attr,
 679         NULL,
 680 };
 681
 682 static umode_t acpi_nfit_dimm_attr_visible(struct kobject *kobj,
 683                 struct attribute *a, int n)
 684 {
 685         struct device *dev = container_of(kobj, struct device, kobj);
 686
 687         if (to_nfit_dcr(dev))
 688                 return a->mode;
 689         else
 690                 return 0;
 691 }
 692
 693 static struct attribute_group acpi_nfit_dimm_attribute_group = {
 694         .name = "nfit",
 695         .attrs = acpi_nfit_dimm_attributes,
 696         .is_visible = acpi_nfit_dimm_attr_visible,
 697 };
 698
 699 static const struct attribute_group *acpi_nfit_dimm_attribute_groups[] = {
 700         &nvdimm_attribute_group,
 701         &nd_device_attribute_group,
 702         &acpi_nfit_dimm_attribute_group,
 703         NULL,
 704 };
 705
 706 static struct nvdimm *acpi_nfit_dimm_by_handle(struct acpi_nfit_desc *acpi_desc,
 707                 u32 device_handle)
 708 {
 709         struct nfit_mem *nfit_mem;
 710
 711         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list)
 712                 if (__to_nfit_memdev(nfit_mem)->device_handle == device_handle)
 713                         return nfit_mem->nvdimm;
 714
 715         return NULL;
 716 }
 717
 718 static int acpi_nfit_add_dimm(struct acpi_nfit_desc *acpi_desc,
 719                 struct nfit_mem *nfit_mem, u32 device_handle)
 720 {
 721         struct acpi_device *adev, *adev_dimm;
 722         struct device *dev = acpi_desc->dev;
 723         const u8 *uuid = to_nfit_uuid(NFIT_DEV_DIMM);
 724         unsigned long long sta;
 725         int i, rc = -ENODEV;
 726         acpi_status status;
 727
 728         nfit_mem->dsm_mask = acpi_desc->dimm_dsm_force_en;
 729         adev = to_acpi_dev(acpi_desc);
 730         if (!adev)
 731                 return 0;
 732
 733         adev_dimm = acpi_find_child_device(adev, device_handle, false);
 734         nfit_mem->adev = adev_dimm;
 735         if (!adev_dimm) {
 736                 dev_err(dev, "no ACPI.NFIT device with _ADR %#x, disabling...\n",
 737                                 device_handle);
 738                 return force_enable_dimms ? 0 : -ENODEV;
 739         }
 740
 741         status = acpi_evaluate_integer(adev_dimm->handle, "_STA", NULL, &sta);
 742         if (status == AE_NOT_FOUND) {
 743                 dev_dbg(dev, "%s missing _STA, assuming enabled...\n",
 744                                 dev_name(&adev_dimm->dev));
 745                 rc = 0;
 746         } else if (ACPI_FAILURE(status))
 747                 dev_err(dev, "%s failed to retrieve_STA, disabling...\n",
 748                                 dev_name(&adev_dimm->dev));
 749         else if ((sta & ACPI_STA_DEVICE_ENABLED) == 0)
 750                 dev_info(dev, "%s disabled by firmware\n",
 751                                 dev_name(&adev_dimm->dev));
 752         else
 753                 rc = 0;
 754
 755         for (i = ND_CMD_SMART; i <= ND_CMD_VENDOR; i++)
 756                 if (acpi_check_dsm(adev_dimm->handle, uuid, 1, 1ULL << i))
 757                         set_bit(i, &nfit_mem->dsm_mask);
 758
 759         return force_enable_dimms ? 0 : rc;
 760 }
 761
 762 static int acpi_nfit_register_dimms(struct acpi_nfit_desc *acpi_desc)
 763 {
 764         struct nfit_mem *nfit_mem;
 765         int dimm_count = 0;
 766
 767         list_for_each_entry(nfit_mem, &acpi_desc->dimms, list) {
 768                 struct nvdimm *nvdimm;
 769                 unsigned long flags = 0;
 770                 u32 device_handle;
 771                 int rc;
 772
 773                 device_handle = __to_nfit_memdev(nfit_mem)->device_handle;
 774                 nvdimm = acpi_nfit_dimm_by_handle(acpi_desc, device_handle);
 775                 if (nvdimm) {
 776                         /*
 777                          * If for some reason we find multiple DCRs the
 778                          * first one wins
 779                          */
 780                         dev_err(acpi_desc->dev, "duplicate DCR detected: %s\n",
 781                                         nvdimm_name(nvdimm));
 782                         continue;
 783                 }
 784
 785                 if (nfit_mem->bdw && nfit_mem->memdev_pmem)
 786                         flags |= NDD_ALIASING;
 787
 788                 rc = acpi_nfit_add_dimm(acpi_desc, nfit_mem, device_handle);
 789                 if (rc)
 790                         continue;
 791
 792                 nvdimm = nvdimm_create(acpi_desc->nvdimm_bus, nfit_mem,
 793                                 acpi_nfit_dimm_attribute_groups,
 794                                 flags, &nfit_mem->dsm_mask);
 795                 if (!nvdimm)
 796                         return -ENOMEM;
 797
 798                 nfit_mem->nvdimm = nvdimm;
 799                 dimm_count++;
 800         }
 801
 802         return nvdimm_bus_check_dimm_count(acpi_desc->nvdimm_bus, dimm_count);
 803 }
 804
 805 static void acpi_nfit_init_dsms(struct acpi_nfit_desc *acpi_desc)
 806 {
 807         struct nvdimm_bus_descriptor *nd_desc = &acpi_desc->nd_desc;
 808         const u8 *uuid = to_nfit_uuid(NFIT_DEV_BUS);
 809         struct acpi_device *adev;
 810         int i;
 811
 812         adev = to_acpi_dev(acpi_desc);
 813         if (!adev)
 814                 return;
 815
 816         for (i = ND_CMD_ARS_CAP; i <= ND_CMD_ARS_STATUS; i++)
 817                 if (acpi_check_dsm(adev->handle, uuid, 1, 1ULL << i))
 818                         set_bit(i, &nd_desc->dsm_mask);
 819 }
 820
 821 static ssize_t range_index_show(struct device *dev,
 822                 struct device_attribute *attr, char *buf)
 823 {
 824         struct nd_region *nd_region = to_nd_region(dev);
 825         struct nfit_spa *nfit_spa = nd_region_provider_data(nd_region);
 826
 827         return sprintf(buf, "%d\n", nfit_spa->spa->range_index);
 828 }
 829 static DEVICE_ATTR_RO(range_index);
 830
 831 static struct attribute *acpi_nfit_region_attributes[] = {
 832         &dev_attr_range_index.attr,
 833         NULL,
 834 };
 835
 836 static struct attribute_group acpi_nfit_region_attribute_group = {
 837         .name = "nfit",
 838         .attrs = acpi_nfit_region_attributes,
 839 };
 840
 841 static const struct attribute_group *acpi_nfit_region_attribute_groups[] = {
 842         &nd_region_attribute_group,
 843         &nd_mapping_attribute_group,
 844         &nd_device_attribute_group,
 845         &acpi_nfit_region_attribute_group,
 846         NULL,
 847 };
 848
 849 /* enough info to uniquely specify an interleave set */
 850 struct nfit_set_info {
 851         struct nfit_set_info_map {
 852                 u64 region_offset;
 853                 u32 serial_number;
 854                 u32 pad;
 855         } mapping[0];
 856 };
 857
 858 static size_t sizeof_nfit_set_info(int num_mappings)
 859 {
 860         return sizeof(struct nfit_set_info)
 861                 + num_mappings * sizeof(struct nfit_set_info_map);
 862 }
 863
 864 static int cmp_map(const void *m0, const void *m1)
 865 {
 866         const struct nfit_set_info_map *map0 = m0;
 867         const struct nfit_set_info_map *map1 = m1;
 868
 869         return memcmp(&map0->region_offset, &map1->region_offset,
 870                         sizeof(u64));
 871 }
 872
 873 /* Retrieve the nth entry referencing this spa */
 874 static struct acpi_nfit_memory_map *memdev_from_spa(
 875                 struct acpi_nfit_desc *acpi_desc, u16 range_index, int n)
 876 {
 877         struct nfit_memdev *nfit_memdev;
 878
 879         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list)
 880                 if (nfit_memdev->memdev->range_index == range_index)
 881                         if (n-- == 0)
 882                                 return nfit_memdev->memdev;
 883         return NULL;
 884 }
 885
 886 static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc,
 887                 struct nd_region_desc *ndr_desc,
 888                 struct acpi_nfit_system_address *spa)
 889 {
 890         int i, spa_type = nfit_spa_type(spa);
 891         struct device *dev = acpi_desc->dev;
 892         struct nd_interleave_set *nd_set;
 893         u16 nr = ndr_desc->num_mappings;
 894         struct nfit_set_info *info;
 895
 896         if (spa_type == NFIT_SPA_PM || spa_type == NFIT_SPA_VOLATILE)
 897                 /* pass */;
 898         else
 899                 return 0;
 900
 901         nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL);
 902         if (!nd_set)
 903                 return -ENOMEM;
 904
 905         info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL);
 906         if (!info)
 907                 return -ENOMEM;
 908         for (i = 0; i < nr; i++) {
 909                 struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
 910                 struct nfit_set_info_map *map = &info->mapping[i];
 911                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
 912                 struct nfit_mem *nfit_mem = nvdimm_provider_data(nvdimm);
 913                 struct acpi_nfit_memory_map *memdev = memdev_from_spa(acpi_desc,
 914                                 spa->range_index, i);
 915
 916                 if (!memdev || !nfit_mem->dcr) {
 917                         dev_err(dev, "%s: failed to find DCR\n", __func__);
 918                         return -ENODEV;
 919                 }
 920
 921                 map->region_offset = memdev->region_offset;
 922                 map->serial_number = nfit_mem->dcr->serial_number;
 923         }
 924
 925         sort(&info->mapping[0], nr, sizeof(struct nfit_set_info_map),
 926                         cmp_map, NULL);
 927         nd_set->cookie = nd_fletcher64(info, sizeof_nfit_set_info(nr), 0);
 928         ndr_desc->nd_set = nd_set;
 929         devm_kfree(dev, info);
 930
 931         return 0;
 932 }
 933
 934 static u64 to_interleave_offset(u64 offset, struct nfit_blk_mmio *mmio)
 935 {
 936         struct acpi_nfit_interleave *idt = mmio->idt;
 937         u32 sub_line_offset, line_index, line_offset;
 938         u64 line_no, table_skip_count, table_offset;
 939
 940         line_no = div_u64_rem(offset, mmio->line_size, &sub_line_offset);
 941         table_skip_count = div_u64_rem(line_no, mmio->num_lines, &line_index);
 942         line_offset = idt->line_offset[line_index]
 943                 * mmio->line_size;
 944         table_offset = table_skip_count * mmio->table_size;
 945
 946         return mmio->base_offset + line_offset + table_offset + sub_line_offset;
 947 }
 948
 949 static u64 read_blk_stat(struct nfit_blk *nfit_blk, unsigned int bw)
 950 {
 951         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
 952         u64 offset = nfit_blk->stat_offset + mmio->size * bw;
 953
 954         if (mmio->num_lines)
 955                 offset = to_interleave_offset(offset, mmio);
 956
 957         return readq(mmio->base + offset);
 958 }
 959
 960 static void write_blk_ctl(struct nfit_blk *nfit_blk, unsigned int bw,
 961                 resource_size_t dpa, unsigned int len, unsigned int write)
 962 {
 963         u64 cmd, offset;
 964         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[DCR];
 965
 966         enum {
 967                 BCW_OFFSET_MASK = (1ULL << 48)-1,
 968                 BCW_LEN_SHIFT = 48,
 969                 BCW_LEN_MASK = (1ULL << 8) - 1,
 970                 BCW_CMD_SHIFT = 56,
 971         };
 972
 973         cmd = (dpa >> L1_CACHE_SHIFT) & BCW_OFFSET_MASK;
 974         len = len >> L1_CACHE_SHIFT;
 975         cmd |= ((u64) len & BCW_LEN_MASK) << BCW_LEN_SHIFT;
 976         cmd |= ((u64) write) << BCW_CMD_SHIFT;
 977
 978         offset = nfit_blk->cmd_offset + mmio->size * bw;
 979         if (mmio->num_lines)
 980                 offset = to_interleave_offset(offset, mmio);
 981
 982         writeq(cmd, mmio->base + offset);
 983         /* FIXME: conditionally perform read-back if mandated by firmware */
 984 }
 985
 986 static int acpi_nfit_blk_single_io(struct nfit_blk *nfit_blk,
 987                 resource_size_t dpa, void *iobuf, size_t len, int rw,
 988                 unsigned int lane)
 989 {
 990         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
 991         unsigned int copied = 0;
 992         u64 base_offset;
 993         int rc;
 994
 995         base_offset = nfit_blk->bdw_offset + dpa % L1_CACHE_BYTES
 996                 + lane * mmio->size;
 997         /* TODO: non-temporal access, flush hints, cache management etc... */
 998         write_blk_ctl(nfit_blk, lane, dpa, len, rw);
 999         while (len) {
1000                 unsigned int c;
1001                 u64 offset;
1002
1003                 if (mmio->num_lines) {
1004                         u32 line_offset;
1005
1006                         offset = to_interleave_offset(base_offset + copied,
1007                                         mmio);
1008                         div_u64_rem(offset, mmio->line_size, &line_offset);
1009                         c = min_t(size_t, len, mmio->line_size - line_offset);
1010                 } else {
1011                         offset = base_offset + nfit_blk->bdw_offset;
1012                         c = len;
1013                 }
1014
1015                 if (rw)
1016                         memcpy(mmio->aperture + offset, iobuf + copied, c);
1017                 else
1018                         memcpy(iobuf + copied, mmio->aperture + offset, c);
1019
1020                 copied += c;
1021                 len -= c;
1022         }
1023         rc = read_blk_stat(nfit_blk, lane) ? -EIO : 0;
1024         return rc;
1025 }
1026
1027 static int acpi_nfit_blk_region_do_io(struct nd_blk_region *ndbr,
1028                 resource_size_t dpa, void *iobuf, u64 len, int rw)
1029 {
1030         struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1031         struct nfit_blk_mmio *mmio = &nfit_blk->mmio[BDW];
1032         struct nd_region *nd_region = nfit_blk->nd_region;
1033         unsigned int lane, copied = 0;
1034         int rc = 0;
1035
1036         lane = nd_region_acquire_lane(nd_region);
1037         while (len) {
1038                 u64 c = min(len, mmio->size);
1039
1040                 rc = acpi_nfit_blk_single_io(nfit_blk, dpa + copied,
1041                                 iobuf + copied, c, rw, lane);
1042                 if (rc)
1043                         break;
1044
1045                 copied += c;
1046                 len -= c;
1047         }
1048         nd_region_release_lane(nd_region, lane);
1049
1050         return rc;
1051 }
1052
1053 static void nfit_spa_mapping_release(struct kref *kref)
1054 {
1055         struct nfit_spa_mapping *spa_map = to_spa_map(kref);
1056         struct acpi_nfit_system_address *spa = spa_map->spa;
1057         struct acpi_nfit_desc *acpi_desc = spa_map->acpi_desc;
1058
1059         WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1060         dev_dbg(acpi_desc->dev, "%s: SPA%d\n", __func__, spa->range_index);
1061         iounmap(spa_map->iomem);
1062         release_mem_region(spa->address, spa->length);
1063         list_del(&spa_map->list);
1064         kfree(spa_map);
1065 }
1066
1067 static struct nfit_spa_mapping *find_spa_mapping(
1068                 struct acpi_nfit_desc *acpi_desc,
1069                 struct acpi_nfit_system_address *spa)
1070 {
1071         struct nfit_spa_mapping *spa_map;
1072
1073         WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1074         list_for_each_entry(spa_map, &acpi_desc->spa_maps, list)
1075                 if (spa_map->spa == spa)
1076                         return spa_map;
1077
1078         return NULL;
1079 }
1080
1081 static void nfit_spa_unmap(struct acpi_nfit_desc *acpi_desc,
1082                 struct acpi_nfit_system_address *spa)
1083 {
1084         struct nfit_spa_mapping *spa_map;
1085
1086         mutex_lock(&acpi_desc->spa_map_mutex);
1087         spa_map = find_spa_mapping(acpi_desc, spa);
1088
1089         if (spa_map)
1090                 kref_put(&spa_map->kref, nfit_spa_mapping_release);
1091         mutex_unlock(&acpi_desc->spa_map_mutex);
1092 }
1093
1094 static void __iomem *__nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
1095                 struct acpi_nfit_system_address *spa)
1096 {
1097         resource_size_t start = spa->address;
1098         resource_size_t n = spa->length;
1099         struct nfit_spa_mapping *spa_map;
1100         struct resource *res;
1101
1102         WARN_ON(!mutex_is_locked(&acpi_desc->spa_map_mutex));
1103
1104         spa_map = find_spa_mapping(acpi_desc, spa);
1105         if (spa_map) {
1106                 kref_get(&spa_map->kref);
1107                 return spa_map->iomem;
1108         }
1109
1110         spa_map = kzalloc(sizeof(*spa_map), GFP_KERNEL);
1111         if (!spa_map)
1112                 return NULL;
1113
1114         INIT_LIST_HEAD(&spa_map->list);
1115         spa_map->spa = spa;
1116         kref_init(&spa_map->kref);
1117         spa_map->acpi_desc = acpi_desc;
1118
1119         res = request_mem_region(start, n, dev_name(acpi_desc->dev));
1120         if (!res)
1121                 goto err_mem;
1122
1123         /* TODO: cacheability based on the spa type */
1124         spa_map->iomem = ioremap_nocache(start, n);
1125         if (!spa_map->iomem)
1126                 goto err_map;
1127
1128         list_add_tail(&spa_map->list, &acpi_desc->spa_maps);
1129         return spa_map->iomem;
1130
1131  err_map:
1132         release_mem_region(start, n);
1133  err_mem:
1134         kfree(spa_map);
1135         return NULL;
1136 }
1137
1138 /**
1139  * nfit_spa_map - interleave-aware managed-mappings of acpi_nfit_system_address ranges
1140  * @nvdimm_bus: NFIT-bus that provided the spa table entry
1141  * @nfit_spa: spa table to map
1142  *
1143  * In the case where block-data-window apertures and
1144  * dimm-control-regions are interleaved they will end up sharing a
1145  * single request_mem_region() + ioremap() for the address range.  In
1146  * the style of devm nfit_spa_map() mappings are automatically dropped
1147  * when all region devices referencing the same mapping are disabled /
1148  * unbound.
1149  */
1150 static void __iomem *nfit_spa_map(struct acpi_nfit_desc *acpi_desc,
1151                 struct acpi_nfit_system_address *spa)
1152 {
1153         void __iomem *iomem;
1154
1155         mutex_lock(&acpi_desc->spa_map_mutex);
1156         iomem = __nfit_spa_map(acpi_desc, spa);
1157         mutex_unlock(&acpi_desc->spa_map_mutex);
1158
1159         return iomem;
1160 }
1161
1162 static int nfit_blk_init_interleave(struct nfit_blk_mmio *mmio,
1163                 struct acpi_nfit_interleave *idt, u16 interleave_ways)
1164 {
1165         if (idt) {
1166                 mmio->num_lines = idt->line_count;
1167                 mmio->line_size = idt->line_size;
1168                 if (interleave_ways == 0)
1169                         return -ENXIO;
1170                 mmio->table_size = mmio->num_lines * interleave_ways
1171                         * mmio->line_size;
1172         }
1173
1174         return 0;
1175 }
1176
1177 static int acpi_nfit_blk_region_enable(struct nvdimm_bus *nvdimm_bus,
1178                 struct device *dev)
1179 {
1180         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1181         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1182         struct nd_blk_region *ndbr = to_nd_blk_region(dev);
1183         struct nfit_blk_mmio *mmio;
1184         struct nfit_blk *nfit_blk;
1185         struct nfit_mem *nfit_mem;
1186         struct nvdimm *nvdimm;
1187         int rc;
1188
1189         nvdimm = nd_blk_region_to_dimm(ndbr);
1190         nfit_mem = nvdimm_provider_data(nvdimm);
1191         if (!nfit_mem || !nfit_mem->dcr || !nfit_mem->bdw) {
1192                 dev_dbg(dev, "%s: missing%s%s%s\n", __func__,
1193                                 nfit_mem ? "" : " nfit_mem",
1194                                 nfit_mem->dcr ? "" : " dcr",
1195                                 nfit_mem->bdw ? "" : " bdw");
1196                 return -ENXIO;
1197         }
1198
1199         nfit_blk = devm_kzalloc(dev, sizeof(*nfit_blk), GFP_KERNEL);
1200         if (!nfit_blk)
1201                 return -ENOMEM;
1202         nd_blk_region_set_provider_data(ndbr, nfit_blk);
1203         nfit_blk->nd_region = to_nd_region(dev);
1204
1205         /* map block aperture memory */
1206         nfit_blk->bdw_offset = nfit_mem->bdw->offset;
1207         mmio = &nfit_blk->mmio[BDW];
1208         mmio->base = nfit_spa_map(acpi_desc, nfit_mem->spa_bdw);
1209         if (!mmio->base) {
1210                 dev_dbg(dev, "%s: %s failed to map bdw\n", __func__,
1211                                 nvdimm_name(nvdimm));
1212                 return -ENOMEM;
1213         }
1214         mmio->size = nfit_mem->bdw->size;
1215         mmio->base_offset = nfit_mem->memdev_bdw->region_offset;
1216         mmio->idt = nfit_mem->idt_bdw;
1217         mmio->spa = nfit_mem->spa_bdw;
1218         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_bdw,
1219                         nfit_mem->memdev_bdw->interleave_ways);
1220         if (rc) {
1221                 dev_dbg(dev, "%s: %s failed to init bdw interleave\n",
1222                                 __func__, nvdimm_name(nvdimm));
1223                 return rc;
1224         }
1225
1226         /* map block control memory */
1227         nfit_blk->cmd_offset = nfit_mem->dcr->command_offset;
1228         nfit_blk->stat_offset = nfit_mem->dcr->status_offset;
1229         mmio = &nfit_blk->mmio[DCR];
1230         mmio->base = nfit_spa_map(acpi_desc, nfit_mem->spa_dcr);
1231         if (!mmio->base) {
1232                 dev_dbg(dev, "%s: %s failed to map dcr\n", __func__,
1233                                 nvdimm_name(nvdimm));
1234                 return -ENOMEM;
1235         }
1236         mmio->size = nfit_mem->dcr->window_size;
1237         mmio->base_offset = nfit_mem->memdev_dcr->region_offset;
1238         mmio->idt = nfit_mem->idt_dcr;
1239         mmio->spa = nfit_mem->spa_dcr;
1240         rc = nfit_blk_init_interleave(mmio, nfit_mem->idt_dcr,
1241                         nfit_mem->memdev_dcr->interleave_ways);
1242         if (rc) {
1243                 dev_dbg(dev, "%s: %s failed to init dcr interleave\n",
1244                                 __func__, nvdimm_name(nvdimm));
1245                 return rc;
1246         }
1247
1248         if (mmio->line_size == 0)
1249                 return 0;
1250
1251         if ((u32) nfit_blk->cmd_offset % mmio->line_size
1252                         + 8 > mmio->line_size) {
1253                 dev_dbg(dev, "cmd_offset crosses interleave boundary\n");
1254                 return -ENXIO;
1255         } else if ((u32) nfit_blk->stat_offset % mmio->line_size
1256                         + 8 > mmio->line_size) {
1257                 dev_dbg(dev, "stat_offset crosses interleave boundary\n");
1258                 return -ENXIO;
1259         }
1260
1261         return 0;
1262 }
1263
1264 static void acpi_nfit_blk_region_disable(struct nvdimm_bus *nvdimm_bus,
1265                 struct device *dev)
1266 {
1267         struct nvdimm_bus_descriptor *nd_desc = to_nd_desc(nvdimm_bus);
1268         struct acpi_nfit_desc *acpi_desc = to_acpi_desc(nd_desc);
1269         struct nd_blk_region *ndbr = to_nd_blk_region(dev);
1270         struct nfit_blk *nfit_blk = nd_blk_region_provider_data(ndbr);
1271         int i;
1272
1273         if (!nfit_blk)
1274                 return; /* never enabled */
1275
1276         /* auto-free BLK spa mappings */
1277         for (i = 0; i < 2; i++) {
1278                 struct nfit_blk_mmio *mmio = &nfit_blk->mmio[i];
1279
1280                 if (mmio->base)
1281                         nfit_spa_unmap(acpi_desc, mmio->spa);
1282         }
1283         nd_blk_region_set_provider_data(ndbr, NULL);
1284         /* devm will free nfit_blk */
1285 }
1286
1287 static int acpi_nfit_init_mapping(struct acpi_nfit_desc *acpi_desc,
1288                 struct nd_mapping *nd_mapping, struct nd_region_desc *ndr_desc,
1289                 struct acpi_nfit_memory_map *memdev,
1290                 struct acpi_nfit_system_address *spa)
1291 {
1292         struct nvdimm *nvdimm = acpi_nfit_dimm_by_handle(acpi_desc,
1293                         memdev->device_handle);
1294         struct nd_blk_region_desc *ndbr_desc;
1295         struct nfit_mem *nfit_mem;
1296         int blk_valid = 0;
1297
1298         if (!nvdimm) {
1299                 dev_err(acpi_desc->dev, "spa%d dimm: %#x not found\n",
1300                                 spa->range_index, memdev->device_handle);
1301                 return -ENODEV;
1302         }
1303
1304         nd_mapping->nvdimm = nvdimm;
1305         switch (nfit_spa_type(spa)) {
1306         case NFIT_SPA_PM:
1307         case NFIT_SPA_VOLATILE:
1308                 nd_mapping->start = memdev->address;
1309                 nd_mapping->size = memdev->region_size;
1310                 break;
1311         case NFIT_SPA_DCR:
1312                 nfit_mem = nvdimm_provider_data(nvdimm);
1313                 if (!nfit_mem || !nfit_mem->bdw) {
1314                         dev_dbg(acpi_desc->dev, "spa%d %s missing bdw\n",
1315                                         spa->range_index, nvdimm_name(nvdimm));
1316                 } else {
1317                         nd_mapping->size = nfit_mem->bdw->capacity;
1318                         nd_mapping->start = nfit_mem->bdw->start_address;
1319                         ndr_desc->num_lanes = nfit_mem->bdw->windows;
1320                         blk_valid = 1;
1321                 }
1322
1323                 ndr_desc->nd_mapping = nd_mapping;
1324                 ndr_desc->num_mappings = blk_valid;
1325                 ndbr_desc = to_blk_region_desc(ndr_desc);
1326                 ndbr_desc->enable = acpi_nfit_blk_region_enable;
1327                 ndbr_desc->disable = acpi_nfit_blk_region_disable;
1328                 ndbr_desc->do_io = acpi_desc->blk_do_io;
1329                 if (!nvdimm_blk_region_create(acpi_desc->nvdimm_bus, ndr_desc))
1330                         return -ENOMEM;
1331                 break;
1332         }
1333
1334         return 0;
1335 }
1336
1337 static int acpi_nfit_register_region(struct acpi_nfit_desc *acpi_desc,
1338                 struct nfit_spa *nfit_spa)
1339 {
1340         static struct nd_mapping nd_mappings[ND_MAX_MAPPINGS];
1341         struct acpi_nfit_system_address *spa = nfit_spa->spa;
1342         struct nd_blk_region_desc ndbr_desc;
1343         struct nd_region_desc *ndr_desc;
1344         struct nfit_memdev *nfit_memdev;
1345         struct nvdimm_bus *nvdimm_bus;
1346         struct resource res;
1347         int count = 0, rc;
1348
1349         if (spa->range_index == 0) {
1350                 dev_dbg(acpi_desc->dev, "%s: detected invalid spa index\n",
1351                                 __func__);
1352                 return 0;
1353         }
1354
1355         memset(&res, 0, sizeof(res));
1356         memset(&nd_mappings, 0, sizeof(nd_mappings));
1357         memset(&ndbr_desc, 0, sizeof(ndbr_desc));
1358         res.start = spa->address;
1359         res.end = res.start + spa->length - 1;
1360         ndr_desc = &ndbr_desc.ndr_desc;
1361         ndr_desc->res = &res;
1362         ndr_desc->provider_data = nfit_spa;
1363         ndr_desc->attr_groups = acpi_nfit_region_attribute_groups;
1364         list_for_each_entry(nfit_memdev, &acpi_desc->memdevs, list) {
1365                 struct acpi_nfit_memory_map *memdev = nfit_memdev->memdev;
1366                 struct nd_mapping *nd_mapping;
1367
1368                 if (memdev->range_index != spa->range_index)
1369                         continue;
1370                 if (count >= ND_MAX_MAPPINGS) {
1371                         dev_err(acpi_desc->dev, "spa%d exceeds max mappings %d\n",
1372                                         spa->range_index, ND_MAX_MAPPINGS);
1373                         return -ENXIO;
1374                 }
1375                 nd_mapping = &nd_mappings[count++];
1376                 rc = acpi_nfit_init_mapping(acpi_desc, nd_mapping, ndr_desc,
1377                                 memdev, spa);
1378                 if (rc)
1379                         return rc;
1380         }
1381
1382         ndr_desc->nd_mapping = nd_mappings;
1383         ndr_desc->num_mappings = count;
1384         rc = acpi_nfit_init_interleave_set(acpi_desc, ndr_desc, spa);
1385         if (rc)
1386                 return rc;
1387
1388         nvdimm_bus = acpi_desc->nvdimm_bus;
1389         if (nfit_spa_type(spa) == NFIT_SPA_PM) {
1390                 if (!nvdimm_pmem_region_create(nvdimm_bus, ndr_desc))
1391                         return -ENOMEM;
1392         } else if (nfit_spa_type(spa) == NFIT_SPA_VOLATILE) {
1393                 if (!nvdimm_volatile_region_create(nvdimm_bus, ndr_desc))
1394                         return -ENOMEM;
1395         }
1396         return 0;
1397 }
1398
1399 static int acpi_nfit_register_regions(struct acpi_nfit_desc *acpi_desc)
1400 {
1401         struct nfit_spa *nfit_spa;
1402
1403         list_for_each_entry(nfit_spa, &acpi_desc->spas, list) {
1404                 int rc = acpi_nfit_register_region(acpi_desc, nfit_spa);
1405
1406                 if (rc)
1407                         return rc;
1408         }
1409         return 0;
1410 }
1411
1412 int acpi_nfit_init(struct acpi_nfit_desc *acpi_desc, acpi_size sz)
1413 {
1414         struct device *dev = acpi_desc->dev;
1415         const void *end;
1416         u8 *data;
1417         int rc;
1418
1419         INIT_LIST_HEAD(&acpi_desc->spa_maps);
1420         INIT_LIST_HEAD(&acpi_desc->spas);
1421         INIT_LIST_HEAD(&acpi_desc->dcrs);
1422         INIT_LIST_HEAD(&acpi_desc->bdws);
1423         INIT_LIST_HEAD(&acpi_desc->idts);
1424         INIT_LIST_HEAD(&acpi_desc->memdevs);
1425         INIT_LIST_HEAD(&acpi_desc->dimms);
1426         mutex_init(&acpi_desc->spa_map_mutex);
1427
1428         data = (u8 *) acpi_desc->nfit;
1429         end = data + sz;
1430         data += sizeof(struct acpi_table_nfit);
1431         while (!IS_ERR_OR_NULL(data))
1432                 data = add_table(acpi_desc, data, end);
1433
1434         if (IS_ERR(data)) {
1435                 dev_dbg(dev, "%s: nfit table parsing error: %ld\n", __func__,
1436                                 PTR_ERR(data));
1437                 return PTR_ERR(data);
1438         }
1439
1440         if (nfit_mem_init(acpi_desc) != 0)
1441                 return -ENOMEM;
1442
1443         acpi_nfit_init_dsms(acpi_desc);
1444
1445         rc = acpi_nfit_register_dimms(acpi_desc);
1446         if (rc)
1447                 return rc;
1448
1449         return acpi_nfit_register_regions(acpi_desc);
1450 }
1451 EXPORT_SYMBOL_GPL(acpi_nfit_init);
1452
1453 static int acpi_nfit_add(struct acpi_device *adev)
1454 {
1455         struct nvdimm_bus_descriptor *nd_desc;
1456         struct acpi_nfit_desc *acpi_desc;
1457         struct device *dev = &adev->dev;
1458         struct acpi_table_header *tbl;
1459         acpi_status status = AE_OK;
1460         acpi_size sz;
1461         int rc;
1462
1463         status = acpi_get_table_with_size("NFIT", 0, &tbl, &sz);
1464         if (ACPI_FAILURE(status)) {
1465                 dev_err(dev, "failed to find NFIT\n");
1466                 return -ENXIO;
1467         }
1468
1469         acpi_desc = devm_kzalloc(dev, sizeof(*acpi_desc), GFP_KERNEL);
1470         if (!acpi_desc)
1471                 return -ENOMEM;
1472
1473         dev_set_drvdata(dev, acpi_desc);
1474         acpi_desc->dev = dev;
1475         acpi_desc->nfit = (struct acpi_table_nfit *) tbl;
1476         acpi_desc->blk_do_io = acpi_nfit_blk_region_do_io;
1477         nd_desc = &acpi_desc->nd_desc;
1478         nd_desc->provider_name = "ACPI.NFIT";
1479         nd_desc->ndctl = acpi_nfit_ctl;
1480         nd_desc->attr_groups = acpi_nfit_attribute_groups;
1481
1482         acpi_desc->nvdimm_bus = nvdimm_bus_register(dev, nd_desc);
1483         if (!acpi_desc->nvdimm_bus)
1484                 return -ENXIO;
1485
1486         rc = acpi_nfit_init(acpi_desc, sz);
1487         if (rc) {
1488                 nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
1489                 return rc;
1490         }
1491         return 0;
1492 }
1493
1494 static int acpi_nfit_remove(struct acpi_device *adev)
1495 {
1496         struct acpi_nfit_desc *acpi_desc = dev_get_drvdata(&adev->dev);
1497
1498         nvdimm_bus_unregister(acpi_desc->nvdimm_bus);
1499         return 0;
1500 }
1501
1502 static const struct acpi_device_id acpi_nfit_ids[] = {
1503         { "ACPI0012", 0 },
1504         { "", 0 },
1505 };
1506 MODULE_DEVICE_TABLE(acpi, acpi_nfit_ids);
1507
1508 static struct acpi_driver acpi_nfit_driver = {
1509         .name = KBUILD_MODNAME,
1510         .ids = acpi_nfit_ids,
1511         .ops = {
1512                 .add = acpi_nfit_add,
1513                 .remove = acpi_nfit_remove,
1514         },
1515 };
1516
1517 static __init int nfit_init(void)
1518 {
1519         BUILD_BUG_ON(sizeof(struct acpi_table_nfit) != 40);
1520         BUILD_BUG_ON(sizeof(struct acpi_nfit_system_address) != 56);
1521         BUILD_BUG_ON(sizeof(struct acpi_nfit_memory_map) != 48);
1522         BUILD_BUG_ON(sizeof(struct acpi_nfit_interleave) != 20);
1523         BUILD_BUG_ON(sizeof(struct acpi_nfit_smbios) != 9);
1524         BUILD_BUG_ON(sizeof(struct acpi_nfit_control_region) != 80);
1525         BUILD_BUG_ON(sizeof(struct acpi_nfit_data_region) != 40);
1526
1527         acpi_str_to_uuid(UUID_VOLATILE_MEMORY, nfit_uuid[NFIT_SPA_VOLATILE]);
1528         acpi_str_to_uuid(UUID_PERSISTENT_MEMORY, nfit_uuid[NFIT_SPA_PM]);
1529         acpi_str_to_uuid(UUID_CONTROL_REGION, nfit_uuid[NFIT_SPA_DCR]);
1530         acpi_str_to_uuid(UUID_DATA_REGION, nfit_uuid[NFIT_SPA_BDW]);
1531         acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_VDISK]);
1532         acpi_str_to_uuid(UUID_VOLATILE_VIRTUAL_CD, nfit_uuid[NFIT_SPA_VCD]);
1533         acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_DISK, nfit_uuid[NFIT_SPA_PDISK]);
1534         acpi_str_to_uuid(UUID_PERSISTENT_VIRTUAL_CD, nfit_uuid[NFIT_SPA_PCD]);
1535         acpi_str_to_uuid(UUID_NFIT_BUS, nfit_uuid[NFIT_DEV_BUS]);
1536         acpi_str_to_uuid(UUID_NFIT_DIMM, nfit_uuid[NFIT_DEV_DIMM]);
1537
1538         return acpi_bus_register_driver(&acpi_nfit_driver);
1539 }
1540
1541 static __exit void nfit_exit(void)
1542 {
1543         acpi_bus_unregister_driver(&acpi_nfit_driver);
1544 }
1545
1546 module_init(nfit_init);
1547 module_exit(nfit_exit);
1548 MODULE_LICENSE("GPL v2");
1549 MODULE_AUTHOR("Intel Corporation");