2 * Copyright (C) 2012 Google, Inc.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
15 #include <linux/device.h>
16 #include <linux/err.h>
17 #include <linux/errno.h>
18 #include <linux/kernel.h>
19 #include <linux/init.h>
21 #include <linux/list.h>
22 #include <linux/memblock.h>
23 #include <linux/rslib.h>
24 #include <linux/slab.h>
25 #include <linux/vmalloc.h>
26 #include <linux/pstore_ram.h>
29 struct persistent_ram_buffer {
36 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */
38 static __initdata LIST_HEAD(persistent_ram_list);
40 static inline size_t buffer_size(struct persistent_ram_zone *prz)
42 return atomic_read(&prz->buffer->size);
45 static inline size_t buffer_start(struct persistent_ram_zone *prz)
47 return atomic_read(&prz->buffer->start);
50 /* increase and wrap the start pointer, returning the old value */
51 static inline size_t buffer_start_add(struct persistent_ram_zone *prz, size_t a)
57 old = atomic_read(&prz->buffer->start);
59 while (unlikely(new > prz->buffer_size))
60 new -= prz->buffer_size;
61 } while (atomic_cmpxchg(&prz->buffer->start, old, new) != old);
66 /* increase the size counter until it hits the max size */
67 static inline void buffer_size_add(struct persistent_ram_zone *prz, size_t a)
72 if (atomic_read(&prz->buffer->size) == prz->buffer_size)
76 old = atomic_read(&prz->buffer->size);
78 if (new > prz->buffer_size)
79 new = prz->buffer_size;
80 } while (atomic_cmpxchg(&prz->buffer->size, old, new) != old);
83 static void notrace persistent_ram_encode_rs8(struct persistent_ram_zone *prz,
84 uint8_t *data, size_t len, uint8_t *ecc)
87 uint16_t par[prz->ecc_size];
89 /* Initialize the parity buffer */
90 memset(par, 0, sizeof(par));
91 encode_rs8(prz->rs_decoder, data, len, par, 0);
92 for (i = 0; i < prz->ecc_size; i++)
96 static int persistent_ram_decode_rs8(struct persistent_ram_zone *prz,
97 void *data, size_t len, uint8_t *ecc)
100 uint16_t par[prz->ecc_size];
102 for (i = 0; i < prz->ecc_size; i++)
104 return decode_rs8(prz->rs_decoder, data, par, len,
105 NULL, 0, NULL, 0, NULL);
108 static void notrace persistent_ram_update_ecc(struct persistent_ram_zone *prz,
109 unsigned int start, unsigned int count)
111 struct persistent_ram_buffer *buffer = prz->buffer;
112 uint8_t *buffer_end = buffer->data + prz->buffer_size;
115 int ecc_block_size = prz->ecc_block_size;
116 int ecc_size = prz->ecc_size;
117 int size = prz->ecc_block_size;
122 block = buffer->data + (start & ~(ecc_block_size - 1));
123 par = prz->par_buffer + (start / ecc_block_size) * prz->ecc_size;
126 if (block + ecc_block_size > buffer_end)
127 size = buffer_end - block;
128 persistent_ram_encode_rs8(prz, block, size, par);
129 block += ecc_block_size;
131 } while (block < buffer->data + start + count);
134 static void persistent_ram_update_header_ecc(struct persistent_ram_zone *prz)
136 struct persistent_ram_buffer *buffer = prz->buffer;
141 persistent_ram_encode_rs8(prz, (uint8_t *)buffer, sizeof(*buffer),
145 static void persistent_ram_ecc_old(struct persistent_ram_zone *prz)
147 struct persistent_ram_buffer *buffer = prz->buffer;
154 block = buffer->data;
155 par = prz->par_buffer;
156 while (block < buffer->data + buffer_size(prz)) {
158 int size = prz->ecc_block_size;
159 if (block + size > buffer->data + prz->buffer_size)
160 size = buffer->data + prz->buffer_size - block;
161 numerr = persistent_ram_decode_rs8(prz, block, size, par);
163 pr_devel("persistent_ram: error in block %p, %d\n",
165 prz->corrected_bytes += numerr;
166 } else if (numerr < 0) {
167 pr_devel("persistent_ram: uncorrectable error in block %p\n",
171 block += prz->ecc_block_size;
172 par += prz->ecc_size;
176 static int persistent_ram_init_ecc(struct persistent_ram_zone *prz,
180 struct persistent_ram_buffer *buffer = prz->buffer;
186 prz->ecc_block_size = 128;
188 prz->ecc_symsize = 8;
189 prz->ecc_poly = 0x11d;
191 ecc_blocks = DIV_ROUND_UP(prz->buffer_size, prz->ecc_block_size);
192 prz->buffer_size -= (ecc_blocks + 1) * prz->ecc_size;
194 if (prz->buffer_size > buffer_size) {
195 pr_err("persistent_ram: invalid size %zu, non-ecc datasize %zu\n",
196 buffer_size, prz->buffer_size);
200 prz->par_buffer = buffer->data + prz->buffer_size;
201 prz->par_header = prz->par_buffer + ecc_blocks * prz->ecc_size;
204 * first consecutive root is 0
205 * primitive element to generate roots = 1
207 prz->rs_decoder = init_rs(prz->ecc_symsize, prz->ecc_poly, 0, 1,
209 if (prz->rs_decoder == NULL) {
210 pr_info("persistent_ram: init_rs failed\n");
214 prz->corrected_bytes = 0;
217 numerr = persistent_ram_decode_rs8(prz, buffer, sizeof(*buffer),
220 pr_info("persistent_ram: error in header, %d\n", numerr);
221 prz->corrected_bytes += numerr;
222 } else if (numerr < 0) {
223 pr_info("persistent_ram: uncorrectable error in header\n");
230 ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz,
231 char *str, size_t len)
235 if (prz->corrected_bytes || prz->bad_blocks)
236 ret = snprintf(str, len, ""
237 "\n%d Corrected bytes, %d unrecoverable blocks\n",
238 prz->corrected_bytes, prz->bad_blocks);
240 ret = snprintf(str, len, "\nNo errors detected\n");
245 static void notrace persistent_ram_update(struct persistent_ram_zone *prz,
246 const void *s, unsigned int start, unsigned int count)
248 struct persistent_ram_buffer *buffer = prz->buffer;
249 memcpy(buffer->data + start, s, count);
250 persistent_ram_update_ecc(prz, start, count);
254 persistent_ram_save_old(struct persistent_ram_zone *prz)
256 struct persistent_ram_buffer *buffer = prz->buffer;
257 size_t size = buffer_size(prz);
258 size_t start = buffer_start(prz);
261 persistent_ram_ecc_old(prz);
263 dest = kmalloc(size, GFP_KERNEL);
265 pr_err("persistent_ram: failed to allocate buffer\n");
270 prz->old_log_size = size;
271 memcpy(prz->old_log, &buffer->data[start], size - start);
272 memcpy(prz->old_log + size - start, &buffer->data[0], start);
275 int notrace persistent_ram_write(struct persistent_ram_zone *prz,
276 const void *s, unsigned int count)
282 if (unlikely(c > prz->buffer_size)) {
283 s += c - prz->buffer_size;
284 c = prz->buffer_size;
287 buffer_size_add(prz, c);
289 start = buffer_start_add(prz, c);
291 rem = prz->buffer_size - start;
292 if (unlikely(rem < c)) {
293 persistent_ram_update(prz, s, start, rem);
298 persistent_ram_update(prz, s, start, c);
300 persistent_ram_update_header_ecc(prz);
305 size_t persistent_ram_old_size(struct persistent_ram_zone *prz)
307 return prz->old_log_size;
310 void *persistent_ram_old(struct persistent_ram_zone *prz)
315 void persistent_ram_free_old(struct persistent_ram_zone *prz)
319 prz->old_log_size = 0;
322 static void *persistent_ram_vmap(phys_addr_t start, size_t size)
325 phys_addr_t page_start;
326 unsigned int page_count;
331 page_start = start - offset_in_page(start);
332 page_count = DIV_ROUND_UP(size + offset_in_page(start), PAGE_SIZE);
334 prot = pgprot_noncached(PAGE_KERNEL);
336 pages = kmalloc(sizeof(struct page *) * page_count, GFP_KERNEL);
338 pr_err("%s: Failed to allocate array for %u pages\n", __func__,
343 for (i = 0; i < page_count; i++) {
344 phys_addr_t addr = page_start + i * PAGE_SIZE;
345 pages[i] = pfn_to_page(addr >> PAGE_SHIFT);
347 vaddr = vmap(pages, page_count, VM_MAP, prot);
353 static void *persistent_ram_iomap(phys_addr_t start, size_t size)
355 if (!request_mem_region(start, size, "persistent_ram")) {
356 pr_err("request mem region (0x%llx@0x%llx) failed\n",
357 (unsigned long long)size, (unsigned long long)start);
361 return ioremap(start, size);
364 static int persistent_ram_buffer_map(phys_addr_t start, phys_addr_t size,
365 struct persistent_ram_zone *prz)
370 if (pfn_valid(start >> PAGE_SHIFT))
371 prz->vaddr = persistent_ram_vmap(start, size);
373 prz->vaddr = persistent_ram_iomap(start, size);
376 pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__,
377 (unsigned long long)size, (unsigned long long)start);
381 prz->buffer = prz->vaddr + offset_in_page(start);
382 prz->buffer_size = size - sizeof(struct persistent_ram_buffer);
387 static int __init persistent_ram_post_init(struct persistent_ram_zone *prz, bool ecc)
393 ret = persistent_ram_init_ecc(prz, prz->buffer_size);
397 if (prz->buffer->sig == PERSISTENT_RAM_SIG) {
398 if (buffer_size(prz) > prz->buffer_size ||
399 buffer_start(prz) > buffer_size(prz))
400 pr_info("persistent_ram: found existing invalid buffer,"
401 " size %zu, start %zu\n",
402 buffer_size(prz), buffer_start(prz));
404 pr_info("persistent_ram: found existing buffer,"
405 " size %zu, start %zu\n",
406 buffer_size(prz), buffer_start(prz));
407 persistent_ram_save_old(prz);
410 pr_info("persistent_ram: no valid data in buffer"
411 " (sig = 0x%08x)\n", prz->buffer->sig);
414 prz->buffer->sig = PERSISTENT_RAM_SIG;
415 atomic_set(&prz->buffer->start, 0);
416 atomic_set(&prz->buffer->size, 0);
421 void persistent_ram_free(struct persistent_ram_zone *prz)
423 if (pfn_valid(prz->paddr >> PAGE_SHIFT)) {
427 release_mem_region(prz->paddr, prz->size);
429 persistent_ram_free_old(prz);
433 struct persistent_ram_zone * __init persistent_ram_new(phys_addr_t start,
437 struct persistent_ram_zone *prz;
440 prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL);
442 pr_err("persistent_ram: failed to allocate persistent ram zone\n");
446 ret = persistent_ram_buffer_map(start, size, prz);
450 persistent_ram_post_init(prz, ecc);
451 persistent_ram_update_header_ecc(prz);
460 static int __init persistent_ram_buffer_init(const char *name,
461 struct persistent_ram_zone *prz)
464 struct persistent_ram *ram;
465 struct persistent_ram_descriptor *desc;
468 list_for_each_entry(ram, &persistent_ram_list, node) {
470 for (i = 0; i < ram->num_descs; i++) {
471 desc = &ram->descs[i];
472 if (!strcmp(desc->name, name))
473 return persistent_ram_buffer_map(start,
483 struct persistent_ram_zone *__persistent_ram_init(struct device *dev, bool ecc)
485 struct persistent_ram_zone *prz;
488 prz = kzalloc(sizeof(struct persistent_ram_zone), GFP_KERNEL);
490 pr_err("persistent_ram: failed to allocate persistent ram zone\n");
494 ret = persistent_ram_buffer_init(dev_name(dev), prz);
496 pr_err("persistent_ram: failed to initialize buffer\n");
500 persistent_ram_post_init(prz, ecc);
508 struct persistent_ram_zone * __init
509 persistent_ram_init_ringbuffer(struct device *dev, bool ecc)
511 return __persistent_ram_init(dev, ecc);
514 int __init persistent_ram_early_init(struct persistent_ram *ram)
518 ret = memblock_reserve(ram->start, ram->size);
520 pr_err("Failed to reserve persistent memory from %08lx-%08lx\n",
521 (long)ram->start, (long)(ram->start + ram->size - 1));
525 list_add_tail(&ram->node, &persistent_ram_list);
527 pr_info("Initialized persistent memory from %08lx-%08lx\n",
528 (long)ram->start, (long)(ram->start + ram->size - 1));