mm: thp: set the accessed flag for old pages on access fault
[pandora-kernel.git] / drivers / idle / intel_idle.c
1 /*
2  * intel_idle.c - native hardware idle loop for modern Intel processors
3  *
4  * Copyright (c) 2010, Intel Corporation.
5  * Len Brown <len.brown@intel.com>
6  *
7  * This program is free software; you can redistribute it and/or modify it
8  * under the terms and conditions of the GNU General Public License,
9  * version 2, as published by the Free Software Foundation.
10  *
11  * This program is distributed in the hope it will be useful, but WITHOUT
12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
14  * more details.
15  *
16  * You should have received a copy of the GNU General Public License along with
17  * this program; if not, write to the Free Software Foundation, Inc.,
18  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19  */
20
21 /*
22  * intel_idle is a cpuidle driver that loads on specific Intel processors
23  * in lieu of the legacy ACPI processor_idle driver.  The intent is to
24  * make Linux more efficient on these processors, as intel_idle knows
25  * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
26  */
27
28 /*
29  * Design Assumptions
30  *
31  * All CPUs have same idle states as boot CPU
32  *
33  * Chipset BM_STS (bus master status) bit is a NOP
34  *      for preventing entry into deep C-stats
35  */
36
37 /*
38  * Known limitations
39  *
40  * The driver currently initializes for_each_online_cpu() upon modprobe.
41  * It it unaware of subsequent processors hot-added to the system.
42  * This means that if you boot with maxcpus=n and later online
43  * processors above n, those processors will use C1 only.
44  *
45  * ACPI has a .suspend hack to turn off deep c-statees during suspend
46  * to avoid complications with the lapic timer workaround.
47  * Have not seen issues with suspend, but may need same workaround here.
48  *
49  * There is currently no kernel-based automatic probing/loading mechanism
50  * if the driver is built as a module.
51  */
52
53 /* un-comment DEBUG to enable pr_debug() statements */
54 #define DEBUG
55
56 #include <linux/kernel.h>
57 #include <linux/cpuidle.h>
58 #include <linux/clockchips.h>
59 #include <linux/hrtimer.h>      /* ktime_get_real() */
60 #include <trace/events/power.h>
61 #include <linux/sched.h>
62 #include <linux/notifier.h>
63 #include <linux/cpu.h>
64 #include <linux/module.h>
65 #include <asm/mwait.h>
66 #include <asm/msr.h>
67
68 #define INTEL_IDLE_VERSION "0.4"
69 #define PREFIX "intel_idle: "
70
71 static struct cpuidle_driver intel_idle_driver = {
72         .name = "intel_idle",
73         .owner = THIS_MODULE,
74 };
75 /* intel_idle.max_cstate=0 disables driver */
76 static int max_cstate = MWAIT_MAX_NUM_CSTATES - 1;
77
78 static unsigned int mwait_substates;
79
80 #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
81 /* Reliable LAPIC Timer States, bit 1 for C1 etc.  */
82 static unsigned int lapic_timer_reliable_states = (1 << 1);      /* Default to only C1 */
83
84 static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
85 static int intel_idle(struct cpuidle_device *dev,
86                         struct cpuidle_driver *drv, int index);
87
88 static struct cpuidle_state *cpuidle_state_table;
89
90 /*
91  * Hardware C-state auto-demotion may not always be optimal.
92  * Indicate which enable bits to clear here.
93  */
94 static unsigned long long auto_demotion_disable_flags;
95
96 /*
97  * Set this flag for states where the HW flushes the TLB for us
98  * and so we don't need cross-calls to keep it consistent.
99  * If this flag is set, SW flushes the TLB, so even if the
100  * HW doesn't do the flushing, this flag is safe to use.
101  */
102 #define CPUIDLE_FLAG_TLB_FLUSHED        0x10000
103
104 /*
105  * States are indexed by the cstate number,
106  * which is also the index into the MWAIT hint array.
107  * Thus C0 is a dummy.
108  */
109 static struct cpuidle_state nehalem_cstates[MWAIT_MAX_NUM_CSTATES] = {
110         { /* MWAIT C0 */ },
111         { /* MWAIT C1 */
112                 .name = "C1-NHM",
113                 .desc = "MWAIT 0x00",
114                 .flags = CPUIDLE_FLAG_TIME_VALID,
115                 .exit_latency = 3,
116                 .target_residency = 6,
117                 .enter = &intel_idle },
118         { /* MWAIT C2 */
119                 .name = "C3-NHM",
120                 .desc = "MWAIT 0x10",
121                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
122                 .exit_latency = 20,
123                 .target_residency = 80,
124                 .enter = &intel_idle },
125         { /* MWAIT C3 */
126                 .name = "C6-NHM",
127                 .desc = "MWAIT 0x20",
128                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
129                 .exit_latency = 200,
130                 .target_residency = 800,
131                 .enter = &intel_idle },
132 };
133
134 static struct cpuidle_state snb_cstates[MWAIT_MAX_NUM_CSTATES] = {
135         { /* MWAIT C0 */ },
136         { /* MWAIT C1 */
137                 .name = "C1-SNB",
138                 .desc = "MWAIT 0x00",
139                 .flags = CPUIDLE_FLAG_TIME_VALID,
140                 .exit_latency = 1,
141                 .target_residency = 1,
142                 .enter = &intel_idle },
143         { /* MWAIT C2 */
144                 .name = "C3-SNB",
145                 .desc = "MWAIT 0x10",
146                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
147                 .exit_latency = 80,
148                 .target_residency = 211,
149                 .enter = &intel_idle },
150         { /* MWAIT C3 */
151                 .name = "C6-SNB",
152                 .desc = "MWAIT 0x20",
153                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
154                 .exit_latency = 104,
155                 .target_residency = 345,
156                 .enter = &intel_idle },
157         { /* MWAIT C4 */
158                 .name = "C7-SNB",
159                 .desc = "MWAIT 0x30",
160                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
161                 .exit_latency = 109,
162                 .target_residency = 345,
163                 .enter = &intel_idle },
164 };
165
166 static struct cpuidle_state ivb_cstates[MWAIT_MAX_NUM_CSTATES] = {
167         { /* MWAIT C0 */ },
168         { /* MWAIT C1 */
169                 .name = "C1-IVB",
170                 .desc = "MWAIT 0x00",
171                 .flags = CPUIDLE_FLAG_TIME_VALID,
172                 .exit_latency = 1,
173                 .target_residency = 1,
174                 .enter = &intel_idle },
175         { /* MWAIT C2 */
176                 .name = "C3-IVB",
177                 .desc = "MWAIT 0x10",
178                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
179                 .exit_latency = 59,
180                 .target_residency = 156,
181                 .enter = &intel_idle },
182         { /* MWAIT C3 */
183                 .name = "C6-IVB",
184                 .desc = "MWAIT 0x20",
185                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
186                 .exit_latency = 80,
187                 .target_residency = 300,
188                 .enter = &intel_idle },
189         { /* MWAIT C4 */
190                 .name = "C7-IVB",
191                 .desc = "MWAIT 0x30",
192                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
193                 .exit_latency = 87,
194                 .target_residency = 300,
195                 .enter = &intel_idle },
196 };
197
198 static struct cpuidle_state atom_cstates[MWAIT_MAX_NUM_CSTATES] = {
199         { /* MWAIT C0 */ },
200         { /* MWAIT C1 */
201                 .name = "C1-ATM",
202                 .desc = "MWAIT 0x00",
203                 .flags = CPUIDLE_FLAG_TIME_VALID,
204                 .exit_latency = 1,
205                 .target_residency = 4,
206                 .enter = &intel_idle },
207         { /* MWAIT C2 */
208                 .name = "C2-ATM",
209                 .desc = "MWAIT 0x10",
210                 .flags = CPUIDLE_FLAG_TIME_VALID,
211                 .exit_latency = 20,
212                 .target_residency = 80,
213                 .enter = &intel_idle },
214         { /* MWAIT C3 */ },
215         { /* MWAIT C4 */
216                 .name = "C4-ATM",
217                 .desc = "MWAIT 0x30",
218                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
219                 .exit_latency = 100,
220                 .target_residency = 400,
221                 .enter = &intel_idle },
222         { /* MWAIT C5 */ },
223         { /* MWAIT C6 */
224                 .name = "C6-ATM",
225                 .desc = "MWAIT 0x52",
226                 .flags = CPUIDLE_FLAG_TIME_VALID | CPUIDLE_FLAG_TLB_FLUSHED,
227                 .exit_latency = 140,
228                 .target_residency = 560,
229                 .enter = &intel_idle },
230 };
231
232 static int get_driver_data(int cstate)
233 {
234         int driver_data;
235         switch (cstate) {
236
237         case 1: /* MWAIT C1 */
238                 driver_data = 0x00;
239                 break;
240         case 2: /* MWAIT C2 */
241                 driver_data = 0x10;
242                 break;
243         case 3: /* MWAIT C3 */
244                 driver_data = 0x20;
245                 break;
246         case 4: /* MWAIT C4 */
247                 driver_data = 0x30;
248                 break;
249         case 5: /* MWAIT C5 */
250                 driver_data = 0x40;
251                 break;
252         case 6: /* MWAIT C6 */
253                 driver_data = 0x52;
254                 break;
255         default:
256                 driver_data = 0x00;
257         }
258         return driver_data;
259 }
260
261 /**
262  * intel_idle
263  * @dev: cpuidle_device
264  * @drv: cpuidle driver
265  * @index: index of cpuidle state
266  *
267  */
268 static int intel_idle(struct cpuidle_device *dev,
269                 struct cpuidle_driver *drv, int index)
270 {
271         unsigned long ecx = 1; /* break on interrupt flag */
272         struct cpuidle_state *state = &drv->states[index];
273         struct cpuidle_state_usage *state_usage = &dev->states_usage[index];
274         unsigned long eax = (unsigned long)cpuidle_get_statedata(state_usage);
275         unsigned int cstate;
276         ktime_t kt_before, kt_after;
277         s64 usec_delta;
278         int cpu = smp_processor_id();
279
280         cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1;
281
282         local_irq_disable();
283
284         /*
285          * leave_mm() to avoid costly and often unnecessary wakeups
286          * for flushing the user TLB's associated with the active mm.
287          */
288         if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
289                 leave_mm(cpu);
290
291         if (!(lapic_timer_reliable_states & (1 << (cstate))))
292                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
293
294         kt_before = ktime_get_real();
295
296         stop_critical_timings();
297         if (!need_resched()) {
298
299                 __monitor((void *)&current_thread_info()->flags, 0, 0);
300                 smp_mb();
301                 if (!need_resched())
302                         __mwait(eax, ecx);
303         }
304
305         start_critical_timings();
306
307         kt_after = ktime_get_real();
308         usec_delta = ktime_to_us(ktime_sub(kt_after, kt_before));
309
310         local_irq_enable();
311
312         if (!(lapic_timer_reliable_states & (1 << (cstate))))
313                 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
314
315         /* Update cpuidle counters */
316         dev->last_residency = (int)usec_delta;
317
318         return index;
319 }
320
321 static void __setup_broadcast_timer(void *arg)
322 {
323         unsigned long reason = (unsigned long)arg;
324         int cpu = smp_processor_id();
325
326         reason = reason ?
327                 CLOCK_EVT_NOTIFY_BROADCAST_ON : CLOCK_EVT_NOTIFY_BROADCAST_OFF;
328
329         clockevents_notify(reason, &cpu);
330 }
331
332 static int setup_broadcast_cpuhp_notify(struct notifier_block *n,
333                 unsigned long action, void *hcpu)
334 {
335         int hotcpu = (unsigned long)hcpu;
336
337         switch (action & 0xf) {
338         case CPU_ONLINE:
339                 smp_call_function_single(hotcpu, __setup_broadcast_timer,
340                         (void *)true, 1);
341                 break;
342         }
343         return NOTIFY_OK;
344 }
345
346 static struct notifier_block setup_broadcast_notifier = {
347         .notifier_call = setup_broadcast_cpuhp_notify,
348 };
349
350 static void auto_demotion_disable(void *dummy)
351 {
352         unsigned long long msr_bits;
353
354         rdmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
355         msr_bits &= ~auto_demotion_disable_flags;
356         wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits);
357 }
358
359 /*
360  * intel_idle_probe()
361  */
362 static int intel_idle_probe(void)
363 {
364         unsigned int eax, ebx, ecx;
365
366         if (max_cstate == 0) {
367                 pr_debug(PREFIX "disabled\n");
368                 return -EPERM;
369         }
370
371         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
372                 return -ENODEV;
373
374         if (!boot_cpu_has(X86_FEATURE_MWAIT))
375                 return -ENODEV;
376
377         if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
378                 return -ENODEV;
379
380         cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
381
382         if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
383             !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
384             !mwait_substates)
385                         return -ENODEV;
386
387         pr_debug(PREFIX "MWAIT substates: 0x%x\n", mwait_substates);
388
389
390         if (boot_cpu_data.x86 != 6)     /* family 6 */
391                 return -ENODEV;
392
393         switch (boot_cpu_data.x86_model) {
394
395         case 0x1A:      /* Core i7, Xeon 5500 series */
396         case 0x1E:      /* Core i7 and i5 Processor - Lynnfield Jasper Forest */
397         case 0x1F:      /* Core i7 and i5 Processor - Nehalem */
398         case 0x2E:      /* Nehalem-EX Xeon */
399         case 0x2F:      /* Westmere-EX Xeon */
400         case 0x25:      /* Westmere */
401         case 0x2C:      /* Westmere */
402                 cpuidle_state_table = nehalem_cstates;
403                 auto_demotion_disable_flags =
404                         (NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE);
405                 break;
406
407         case 0x1C:      /* 28 - Atom Processor */
408                 cpuidle_state_table = atom_cstates;
409                 break;
410
411         case 0x26:      /* 38 - Lincroft Atom Processor */
412                 cpuidle_state_table = atom_cstates;
413                 auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE;
414                 break;
415
416         case 0x2A:      /* SNB */
417         case 0x2D:      /* SNB Xeon */
418                 cpuidle_state_table = snb_cstates;
419                 break;
420
421         case 0x3A:      /* IVB */
422         case 0x3E:      /* IVB Xeon */
423                 cpuidle_state_table = ivb_cstates;
424                 break;
425
426         default:
427                 pr_debug(PREFIX "does not run on family %d model %d\n",
428                         boot_cpu_data.x86, boot_cpu_data.x86_model);
429                 return -ENODEV;
430         }
431
432         if (boot_cpu_has(X86_FEATURE_ARAT))     /* Always Reliable APIC Timer */
433                 lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
434         else
435                 on_each_cpu(__setup_broadcast_timer, (void *)true, 1);
436
437         pr_debug(PREFIX "v" INTEL_IDLE_VERSION
438                 " model 0x%X\n", boot_cpu_data.x86_model);
439
440         pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n",
441                 lapic_timer_reliable_states);
442         return 0;
443 }
444
445 /*
446  * intel_idle_cpuidle_devices_uninit()
447  * unregister, free cpuidle_devices
448  */
449 static void intel_idle_cpuidle_devices_uninit(void)
450 {
451         int i;
452         struct cpuidle_device *dev;
453
454         for_each_online_cpu(i) {
455                 dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
456                 cpuidle_unregister_device(dev);
457         }
458
459         free_percpu(intel_idle_cpuidle_devices);
460         return;
461 }
462 /*
463  * intel_idle_cpuidle_driver_init()
464  * allocate, initialize cpuidle_states
465  */
466 static int intel_idle_cpuidle_driver_init(void)
467 {
468         int cstate;
469         struct cpuidle_driver *drv = &intel_idle_driver;
470
471         drv->state_count = 1;
472
473         for (cstate = 1; cstate < MWAIT_MAX_NUM_CSTATES; ++cstate) {
474                 int num_substates;
475
476                 if (cstate > max_cstate) {
477                         printk(PREFIX "max_cstate %d reached\n",
478                                 max_cstate);
479                         break;
480                 }
481
482                 /* does the state exist in CPUID.MWAIT? */
483                 num_substates = (mwait_substates >> ((cstate) * 4))
484                                         & MWAIT_SUBSTATE_MASK;
485                 if (num_substates == 0)
486                         continue;
487                 /* is the state not enabled? */
488                 if (cpuidle_state_table[cstate].enter == NULL) {
489                         /* does the driver not know about the state? */
490                         if (*cpuidle_state_table[cstate].name == '\0')
491                                 pr_debug(PREFIX "unaware of model 0x%x"
492                                         " MWAIT %d please"
493                                         " contact lenb@kernel.org",
494                                 boot_cpu_data.x86_model, cstate);
495                         continue;
496                 }
497
498                 if ((cstate > 2) &&
499                         !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
500                         mark_tsc_unstable("TSC halts in idle"
501                                         " states deeper than C2");
502
503                 drv->states[drv->state_count] = /* structure copy */
504                         cpuidle_state_table[cstate];
505
506                 drv->state_count += 1;
507         }
508
509         if (auto_demotion_disable_flags)
510                 on_each_cpu(auto_demotion_disable, NULL, 1);
511
512         return 0;
513 }
514
515
516 /*
517  * intel_idle_cpuidle_devices_init()
518  * allocate, initialize, register cpuidle_devices
519  */
520 static int intel_idle_cpuidle_devices_init(void)
521 {
522         int i, cstate;
523         struct cpuidle_device *dev;
524
525         intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
526         if (intel_idle_cpuidle_devices == NULL)
527                 return -ENOMEM;
528
529         for_each_online_cpu(i) {
530                 dev = per_cpu_ptr(intel_idle_cpuidle_devices, i);
531
532                 dev->state_count = 1;
533
534                 for (cstate = 1; cstate < MWAIT_MAX_NUM_CSTATES; ++cstate) {
535                         int num_substates;
536
537                         if (cstate > max_cstate) {
538                                 printk(PREFIX "max_cstate %d reached\n",
539                                         max_cstate);
540                                 break;
541                         }
542
543                         /* does the state exist in CPUID.MWAIT? */
544                         num_substates = (mwait_substates >> ((cstate) * 4))
545                                                 & MWAIT_SUBSTATE_MASK;
546                         if (num_substates == 0)
547                                 continue;
548                         /* is the state not enabled? */
549                         if (cpuidle_state_table[cstate].enter == NULL) {
550                                 continue;
551                         }
552
553                         dev->states_usage[dev->state_count].driver_data =
554                                 (void *)get_driver_data(cstate);
555
556                         dev->state_count += 1;
557                 }
558
559                 dev->cpu = i;
560                 if (cpuidle_register_device(dev)) {
561                         pr_debug(PREFIX "cpuidle_register_device %d failed!\n",
562                                  i);
563                         intel_idle_cpuidle_devices_uninit();
564                         return -EIO;
565                 }
566         }
567
568         return 0;
569 }
570
571
572 static int __init intel_idle_init(void)
573 {
574         int retval;
575
576         /* Do not load intel_idle at all for now if idle= is passed */
577         if (boot_option_idle_override != IDLE_NO_OVERRIDE)
578                 return -ENODEV;
579
580         retval = intel_idle_probe();
581         if (retval)
582                 return retval;
583
584         intel_idle_cpuidle_driver_init();
585         retval = cpuidle_register_driver(&intel_idle_driver);
586         if (retval) {
587                 printk(KERN_DEBUG PREFIX "intel_idle yielding to %s",
588                         cpuidle_get_driver()->name);
589                 return retval;
590         }
591
592         retval = intel_idle_cpuidle_devices_init();
593         if (retval) {
594                 cpuidle_unregister_driver(&intel_idle_driver);
595                 return retval;
596         }
597
598         if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE)
599                 register_cpu_notifier(&setup_broadcast_notifier);
600
601         return 0;
602 }
603
604 static void __exit intel_idle_exit(void)
605 {
606         intel_idle_cpuidle_devices_uninit();
607         cpuidle_unregister_driver(&intel_idle_driver);
608
609         if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) {
610                 on_each_cpu(__setup_broadcast_timer, (void *)false, 1);
611                 unregister_cpu_notifier(&setup_broadcast_notifier);
612         }
613
614         return;
615 }
616
617 module_init(intel_idle_init);
618 module_exit(intel_idle_exit);
619
620 module_param(max_cstate, int, 0444);
621
622 MODULE_AUTHOR("Len Brown <len.brown@intel.com>");
623 MODULE_DESCRIPTION("Cpuidle driver for Intel Hardware v" INTEL_IDLE_VERSION);
624 MODULE_LICENSE("GPL");