2 * linux/drivers/cpufreq/cpufreq.c
4 * Copyright (C) 2001 Russell King
5 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
6 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
8 * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
9 * Added handling for CPU hotplug
10 * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
11 * Fix handling for CPU hotplug -- affected CPUs
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License version 2 as
15 * published by the Free Software Foundation.
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20 #include <linux/cpu.h>
21 #include <linux/cpufreq.h>
22 #include <linux/delay.h>
23 #include <linux/device.h>
24 #include <linux/init.h>
25 #include <linux/kernel_stat.h>
26 #include <linux/module.h>
27 #include <linux/mutex.h>
28 #include <linux/slab.h>
29 #include <linux/suspend.h>
30 #include <linux/tick.h>
31 #include <trace/events/power.h>
34 * The "cpufreq driver" - the arch- or hardware-dependent low
35 * level driver of CPUFreq support, and its spinlock. This lock
36 * also protects the cpufreq_cpu_data array.
38 static struct cpufreq_driver *cpufreq_driver;
39 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
40 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data_fallback);
41 static DEFINE_RWLOCK(cpufreq_driver_lock);
42 DEFINE_MUTEX(cpufreq_governor_lock);
43 static LIST_HEAD(cpufreq_policy_list);
45 /* This one keeps track of the previously set governor of a removed CPU */
46 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
48 /* Flag to suspend/resume CPUFreq governors */
49 static bool cpufreq_suspended;
51 static inline bool has_target(void)
53 return cpufreq_driver->target_index || cpufreq_driver->target;
57 * rwsem to guarantee that cpufreq driver module doesn't unload during critical
60 static DECLARE_RWSEM(cpufreq_rwsem);
62 /* internal prototypes */
63 static int __cpufreq_governor(struct cpufreq_policy *policy,
65 static unsigned int __cpufreq_get(unsigned int cpu);
66 static void handle_update(struct work_struct *work);
69 * Two notifier lists: the "policy" list is involved in the
70 * validation process for a new CPU frequency policy; the
71 * "transition" list for kernel code that needs to handle
72 * changes to devices when the CPU clock speed changes.
73 * The mutex locks both lists.
75 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
76 static struct srcu_notifier_head cpufreq_transition_notifier_list;
78 static bool init_cpufreq_transition_notifier_list_called;
79 static int __init init_cpufreq_transition_notifier_list(void)
81 srcu_init_notifier_head(&cpufreq_transition_notifier_list);
82 init_cpufreq_transition_notifier_list_called = true;
85 pure_initcall(init_cpufreq_transition_notifier_list);
87 static int off __read_mostly;
88 static int cpufreq_disabled(void)
92 void disable_cpufreq(void)
96 static LIST_HEAD(cpufreq_governor_list);
97 static DEFINE_MUTEX(cpufreq_governor_mutex);
99 bool have_governor_per_policy(void)
101 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
103 EXPORT_SYMBOL_GPL(have_governor_per_policy);
105 struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
107 if (have_governor_per_policy())
108 return &policy->kobj;
110 return cpufreq_global_kobject;
112 EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
114 static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
120 cur_wall_time = jiffies64_to_cputime64(get_jiffies_64());
122 busy_time = kcpustat_cpu(cpu).cpustat[CPUTIME_USER];
123 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SYSTEM];
124 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_IRQ];
125 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_SOFTIRQ];
126 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_STEAL];
127 busy_time += kcpustat_cpu(cpu).cpustat[CPUTIME_NICE];
129 idle_time = cur_wall_time - busy_time;
131 *wall = cputime_to_usecs(cur_wall_time);
133 return cputime_to_usecs(idle_time);
136 u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
138 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
140 if (idle_time == -1ULL)
141 return get_cpu_idle_time_jiffy(cpu, wall);
143 idle_time += get_cpu_iowait_time_us(cpu, wall);
147 EXPORT_SYMBOL_GPL(get_cpu_idle_time);
150 * This is a generic cpufreq init() routine which can be used by cpufreq
151 * drivers of SMP systems. It will do following:
152 * - validate & show freq table passed
153 * - set policies transition latency
154 * - policy->cpus with all possible CPUs
156 int cpufreq_generic_init(struct cpufreq_policy *policy,
157 struct cpufreq_frequency_table *table,
158 unsigned int transition_latency)
162 ret = cpufreq_table_validate_and_show(policy, table);
164 pr_err("%s: invalid frequency table: %d\n", __func__, ret);
168 policy->cpuinfo.transition_latency = transition_latency;
171 * The driver only supports the SMP configuartion where all processors
172 * share the clock and voltage and clock.
174 cpumask_setall(policy->cpus);
178 EXPORT_SYMBOL_GPL(cpufreq_generic_init);
180 unsigned int cpufreq_generic_get(unsigned int cpu)
182 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
184 if (!policy || IS_ERR(policy->clk)) {
185 pr_err("%s: No %s associated to cpu: %d\n",
186 __func__, policy ? "clk" : "policy", cpu);
190 return clk_get_rate(policy->clk) / 1000;
192 EXPORT_SYMBOL_GPL(cpufreq_generic_get);
194 /* Only for cpufreq core internal use */
195 struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
197 return per_cpu(cpufreq_cpu_data, cpu);
200 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
202 struct cpufreq_policy *policy = NULL;
205 if (cpufreq_disabled() || (cpu >= nr_cpu_ids))
208 if (!down_read_trylock(&cpufreq_rwsem))
211 /* get the cpufreq driver */
212 read_lock_irqsave(&cpufreq_driver_lock, flags);
214 if (cpufreq_driver) {
216 policy = per_cpu(cpufreq_cpu_data, cpu);
218 kobject_get(&policy->kobj);
221 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
224 up_read(&cpufreq_rwsem);
228 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
230 void cpufreq_cpu_put(struct cpufreq_policy *policy)
232 if (cpufreq_disabled())
235 kobject_put(&policy->kobj);
236 up_read(&cpufreq_rwsem);
238 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
240 /*********************************************************************
241 * EXTERNALLY AFFECTING FREQUENCY CHANGES *
242 *********************************************************************/
245 * adjust_jiffies - adjust the system "loops_per_jiffy"
247 * This function alters the system "loops_per_jiffy" for the clock
248 * speed change. Note that loops_per_jiffy cannot be updated on SMP
249 * systems as each CPU might be scaled differently. So, use the arch
250 * per-CPU loops_per_jiffy value wherever possible.
253 static unsigned long l_p_j_ref;
254 static unsigned int l_p_j_ref_freq;
256 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
258 if (ci->flags & CPUFREQ_CONST_LOOPS)
261 if (!l_p_j_ref_freq) {
262 l_p_j_ref = loops_per_jiffy;
263 l_p_j_ref_freq = ci->old;
264 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
265 l_p_j_ref, l_p_j_ref_freq);
267 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
268 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
270 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
271 loops_per_jiffy, ci->new);
275 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
281 static void __cpufreq_notify_transition(struct cpufreq_policy *policy,
282 struct cpufreq_freqs *freqs, unsigned int state)
284 BUG_ON(irqs_disabled());
286 if (cpufreq_disabled())
289 freqs->flags = cpufreq_driver->flags;
290 pr_debug("notification %u of frequency transition to %u kHz\n",
295 case CPUFREQ_PRECHANGE:
296 /* detect if the driver reported a value as "old frequency"
297 * which is not equal to what the cpufreq core thinks is
300 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
301 if ((policy) && (policy->cpu == freqs->cpu) &&
302 (policy->cur) && (policy->cur != freqs->old)) {
303 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
304 freqs->old, policy->cur);
305 freqs->old = policy->cur;
308 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
309 CPUFREQ_PRECHANGE, freqs);
310 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
313 case CPUFREQ_POSTCHANGE:
314 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
315 pr_debug("FREQ: %lu - CPU: %lu\n",
316 (unsigned long)freqs->new, (unsigned long)freqs->cpu);
317 trace_cpu_frequency(freqs->new, freqs->cpu);
318 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
319 CPUFREQ_POSTCHANGE, freqs);
320 if (likely(policy) && likely(policy->cpu == freqs->cpu))
321 policy->cur = freqs->new;
327 * cpufreq_notify_transition - call notifier chain and adjust_jiffies
328 * on frequency transition.
330 * This function calls the transition notifiers and the "adjust_jiffies"
331 * function. It is called twice on all CPU frequency changes that have
334 void cpufreq_notify_transition(struct cpufreq_policy *policy,
335 struct cpufreq_freqs *freqs, unsigned int state)
337 for_each_cpu(freqs->cpu, policy->cpus)
338 __cpufreq_notify_transition(policy, freqs, state);
340 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
342 /* Do post notifications when there are chances that transition has failed */
343 void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
344 struct cpufreq_freqs *freqs, int transition_failed)
346 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
347 if (!transition_failed)
350 swap(freqs->old, freqs->new);
351 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
352 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
354 EXPORT_SYMBOL_GPL(cpufreq_notify_post_transition);
356 void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
357 struct cpufreq_freqs *freqs)
360 wait_event(policy->transition_wait, !policy->transition_ongoing);
362 spin_lock(&policy->transition_lock);
364 if (unlikely(policy->transition_ongoing)) {
365 spin_unlock(&policy->transition_lock);
369 policy->transition_ongoing = true;
371 spin_unlock(&policy->transition_lock);
373 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
375 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
377 void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
378 struct cpufreq_freqs *freqs, int transition_failed)
380 if (unlikely(WARN_ON(!policy->transition_ongoing)))
383 cpufreq_notify_post_transition(policy, freqs, transition_failed);
385 policy->transition_ongoing = false;
387 wake_up(&policy->transition_wait);
389 EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
392 /*********************************************************************
394 *********************************************************************/
395 static ssize_t show_boost(struct kobject *kobj,
396 struct attribute *attr, char *buf)
398 return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
401 static ssize_t store_boost(struct kobject *kobj, struct attribute *attr,
402 const char *buf, size_t count)
406 ret = sscanf(buf, "%d", &enable);
407 if (ret != 1 || enable < 0 || enable > 1)
410 if (cpufreq_boost_trigger_state(enable)) {
411 pr_err("%s: Cannot %s BOOST!\n",
412 __func__, enable ? "enable" : "disable");
416 pr_debug("%s: cpufreq BOOST %s\n",
417 __func__, enable ? "enabled" : "disabled");
421 define_one_global_rw(boost);
423 static struct cpufreq_governor *__find_governor(const char *str_governor)
425 struct cpufreq_governor *t;
427 list_for_each_entry(t, &cpufreq_governor_list, governor_list)
428 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
435 * cpufreq_parse_governor - parse a governor string
437 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
438 struct cpufreq_governor **governor)
445 if (cpufreq_driver->setpolicy) {
446 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
447 *policy = CPUFREQ_POLICY_PERFORMANCE;
449 } else if (!strnicmp(str_governor, "powersave",
451 *policy = CPUFREQ_POLICY_POWERSAVE;
454 } else if (has_target()) {
455 struct cpufreq_governor *t;
457 mutex_lock(&cpufreq_governor_mutex);
459 t = __find_governor(str_governor);
464 mutex_unlock(&cpufreq_governor_mutex);
465 ret = request_module("cpufreq_%s", str_governor);
466 mutex_lock(&cpufreq_governor_mutex);
469 t = __find_governor(str_governor);
477 mutex_unlock(&cpufreq_governor_mutex);
484 * cpufreq_per_cpu_attr_read() / show_##file_name() -
485 * print out cpufreq information
487 * Write out information from cpufreq_driver->policy[cpu]; object must be
491 #define show_one(file_name, object) \
492 static ssize_t show_##file_name \
493 (struct cpufreq_policy *policy, char *buf) \
495 return sprintf(buf, "%u\n", policy->object); \
498 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
499 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
500 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
501 show_one(scaling_min_freq, min);
502 show_one(scaling_max_freq, max);
503 show_one(scaling_cur_freq, cur);
505 static int cpufreq_set_policy(struct cpufreq_policy *policy,
506 struct cpufreq_policy *new_policy);
509 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
511 #define store_one(file_name, object) \
512 static ssize_t store_##file_name \
513 (struct cpufreq_policy *policy, const char *buf, size_t count) \
516 struct cpufreq_policy new_policy; \
518 ret = cpufreq_get_policy(&new_policy, policy->cpu); \
522 ret = sscanf(buf, "%u", &new_policy.object); \
526 ret = cpufreq_set_policy(policy, &new_policy); \
527 policy->user_policy.object = policy->object; \
529 return ret ? ret : count; \
532 store_one(scaling_min_freq, min);
533 store_one(scaling_max_freq, max);
536 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
538 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
541 unsigned int cur_freq = __cpufreq_get(policy->cpu);
543 return sprintf(buf, "<unknown>");
544 return sprintf(buf, "%u\n", cur_freq);
548 * show_scaling_governor - show the current policy for the specified CPU
550 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
552 if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
553 return sprintf(buf, "powersave\n");
554 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
555 return sprintf(buf, "performance\n");
556 else if (policy->governor)
557 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
558 policy->governor->name);
563 * store_scaling_governor - store policy for the specified CPU
565 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
566 const char *buf, size_t count)
569 char str_governor[16];
570 struct cpufreq_policy new_policy;
572 ret = cpufreq_get_policy(&new_policy, policy->cpu);
576 ret = sscanf(buf, "%15s", str_governor);
580 if (cpufreq_parse_governor(str_governor, &new_policy.policy,
581 &new_policy.governor))
584 ret = cpufreq_set_policy(policy, &new_policy);
586 policy->user_policy.policy = policy->policy;
587 policy->user_policy.governor = policy->governor;
596 * show_scaling_driver - show the cpufreq driver currently loaded
598 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
600 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
604 * show_scaling_available_governors - show the available CPUfreq governors
606 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
610 struct cpufreq_governor *t;
613 i += sprintf(buf, "performance powersave");
617 list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
618 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
619 - (CPUFREQ_NAME_LEN + 2)))
621 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
624 i += sprintf(&buf[i], "\n");
628 ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
633 for_each_cpu(cpu, mask) {
635 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
636 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
637 if (i >= (PAGE_SIZE - 5))
640 i += sprintf(&buf[i], "\n");
643 EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
646 * show_related_cpus - show the CPUs affected by each transition even if
647 * hw coordination is in use
649 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
651 return cpufreq_show_cpus(policy->related_cpus, buf);
655 * show_affected_cpus - show the CPUs affected by each transition
657 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
659 return cpufreq_show_cpus(policy->cpus, buf);
662 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
663 const char *buf, size_t count)
665 unsigned int freq = 0;
668 if (!policy->governor || !policy->governor->store_setspeed)
671 ret = sscanf(buf, "%u", &freq);
675 policy->governor->store_setspeed(policy, freq);
680 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
682 if (!policy->governor || !policy->governor->show_setspeed)
683 return sprintf(buf, "<unsupported>\n");
685 return policy->governor->show_setspeed(policy, buf);
689 * show_bios_limit - show the current cpufreq HW/BIOS limitation
691 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
695 if (cpufreq_driver->bios_limit) {
696 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
698 return sprintf(buf, "%u\n", limit);
700 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
703 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
704 cpufreq_freq_attr_ro(cpuinfo_min_freq);
705 cpufreq_freq_attr_ro(cpuinfo_max_freq);
706 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
707 cpufreq_freq_attr_ro(scaling_available_governors);
708 cpufreq_freq_attr_ro(scaling_driver);
709 cpufreq_freq_attr_ro(scaling_cur_freq);
710 cpufreq_freq_attr_ro(bios_limit);
711 cpufreq_freq_attr_ro(related_cpus);
712 cpufreq_freq_attr_ro(affected_cpus);
713 cpufreq_freq_attr_rw(scaling_min_freq);
714 cpufreq_freq_attr_rw(scaling_max_freq);
715 cpufreq_freq_attr_rw(scaling_governor);
716 cpufreq_freq_attr_rw(scaling_setspeed);
718 static struct attribute *default_attrs[] = {
719 &cpuinfo_min_freq.attr,
720 &cpuinfo_max_freq.attr,
721 &cpuinfo_transition_latency.attr,
722 &scaling_min_freq.attr,
723 &scaling_max_freq.attr,
726 &scaling_governor.attr,
727 &scaling_driver.attr,
728 &scaling_available_governors.attr,
729 &scaling_setspeed.attr,
733 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
734 #define to_attr(a) container_of(a, struct freq_attr, attr)
736 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
738 struct cpufreq_policy *policy = to_policy(kobj);
739 struct freq_attr *fattr = to_attr(attr);
742 if (!down_read_trylock(&cpufreq_rwsem))
745 down_read(&policy->rwsem);
748 ret = fattr->show(policy, buf);
752 up_read(&policy->rwsem);
753 up_read(&cpufreq_rwsem);
758 static ssize_t store(struct kobject *kobj, struct attribute *attr,
759 const char *buf, size_t count)
761 struct cpufreq_policy *policy = to_policy(kobj);
762 struct freq_attr *fattr = to_attr(attr);
763 ssize_t ret = -EINVAL;
767 if (!cpu_online(policy->cpu))
770 if (!down_read_trylock(&cpufreq_rwsem))
773 down_write(&policy->rwsem);
776 ret = fattr->store(policy, buf, count);
780 up_write(&policy->rwsem);
782 up_read(&cpufreq_rwsem);
789 static void cpufreq_sysfs_release(struct kobject *kobj)
791 struct cpufreq_policy *policy = to_policy(kobj);
792 pr_debug("last reference is dropped\n");
793 complete(&policy->kobj_unregister);
796 static const struct sysfs_ops sysfs_ops = {
801 static struct kobj_type ktype_cpufreq = {
802 .sysfs_ops = &sysfs_ops,
803 .default_attrs = default_attrs,
804 .release = cpufreq_sysfs_release,
807 struct kobject *cpufreq_global_kobject;
808 EXPORT_SYMBOL(cpufreq_global_kobject);
810 static int cpufreq_global_kobject_usage;
812 int cpufreq_get_global_kobject(void)
814 if (!cpufreq_global_kobject_usage++)
815 return kobject_add(cpufreq_global_kobject,
816 &cpu_subsys.dev_root->kobj, "%s", "cpufreq");
820 EXPORT_SYMBOL(cpufreq_get_global_kobject);
822 void cpufreq_put_global_kobject(void)
824 if (!--cpufreq_global_kobject_usage)
825 kobject_del(cpufreq_global_kobject);
827 EXPORT_SYMBOL(cpufreq_put_global_kobject);
829 int cpufreq_sysfs_create_file(const struct attribute *attr)
831 int ret = cpufreq_get_global_kobject();
834 ret = sysfs_create_file(cpufreq_global_kobject, attr);
836 cpufreq_put_global_kobject();
841 EXPORT_SYMBOL(cpufreq_sysfs_create_file);
843 void cpufreq_sysfs_remove_file(const struct attribute *attr)
845 sysfs_remove_file(cpufreq_global_kobject, attr);
846 cpufreq_put_global_kobject();
848 EXPORT_SYMBOL(cpufreq_sysfs_remove_file);
850 /* symlink affected CPUs */
851 static int cpufreq_add_dev_symlink(struct cpufreq_policy *policy)
856 for_each_cpu(j, policy->cpus) {
857 struct device *cpu_dev;
859 if (j == policy->cpu)
862 pr_debug("Adding link for CPU: %u\n", j);
863 cpu_dev = get_cpu_device(j);
864 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
872 static int cpufreq_add_dev_interface(struct cpufreq_policy *policy,
875 struct freq_attr **drv_attr;
878 /* prepare interface data */
879 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
880 &dev->kobj, "cpufreq");
884 /* set up files for this cpu device */
885 drv_attr = cpufreq_driver->attr;
886 while ((drv_attr) && (*drv_attr)) {
887 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
889 goto err_out_kobj_put;
892 if (cpufreq_driver->get) {
893 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
895 goto err_out_kobj_put;
898 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
900 goto err_out_kobj_put;
902 if (cpufreq_driver->bios_limit) {
903 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
905 goto err_out_kobj_put;
908 ret = cpufreq_add_dev_symlink(policy);
910 goto err_out_kobj_put;
915 kobject_put(&policy->kobj);
916 wait_for_completion(&policy->kobj_unregister);
920 static void cpufreq_init_policy(struct cpufreq_policy *policy)
922 struct cpufreq_governor *gov = NULL;
923 struct cpufreq_policy new_policy;
926 memcpy(&new_policy, policy, sizeof(*policy));
928 /* Update governor of new_policy to the governor used before hotplug */
929 gov = __find_governor(per_cpu(cpufreq_cpu_governor, policy->cpu));
931 pr_debug("Restoring governor %s for cpu %d\n",
932 policy->governor->name, policy->cpu);
934 gov = CPUFREQ_DEFAULT_GOVERNOR;
936 new_policy.governor = gov;
938 /* Use the default policy if its valid. */
939 if (cpufreq_driver->setpolicy)
940 cpufreq_parse_governor(gov->name, &new_policy.policy, NULL);
942 /* set default policy */
943 ret = cpufreq_set_policy(policy, &new_policy);
945 pr_debug("setting policy failed\n");
946 if (cpufreq_driver->exit)
947 cpufreq_driver->exit(policy);
951 #ifdef CONFIG_HOTPLUG_CPU
952 static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy,
953 unsigned int cpu, struct device *dev)
959 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
961 pr_err("%s: Failed to stop governor\n", __func__);
966 down_write(&policy->rwsem);
968 write_lock_irqsave(&cpufreq_driver_lock, flags);
970 cpumask_set_cpu(cpu, policy->cpus);
971 per_cpu(cpufreq_cpu_data, cpu) = policy;
972 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
974 up_write(&policy->rwsem);
977 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
979 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
982 pr_err("%s: Failed to start governor\n", __func__);
987 return sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq");
991 static struct cpufreq_policy *cpufreq_policy_restore(unsigned int cpu)
993 struct cpufreq_policy *policy;
996 read_lock_irqsave(&cpufreq_driver_lock, flags);
998 policy = per_cpu(cpufreq_cpu_data_fallback, cpu);
1000 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1002 policy->governor = NULL;
1007 static struct cpufreq_policy *cpufreq_policy_alloc(void)
1009 struct cpufreq_policy *policy;
1011 policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1015 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1016 goto err_free_policy;
1018 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1019 goto err_free_cpumask;
1021 INIT_LIST_HEAD(&policy->policy_list);
1022 init_rwsem(&policy->rwsem);
1023 spin_lock_init(&policy->transition_lock);
1024 init_waitqueue_head(&policy->transition_wait);
1029 free_cpumask_var(policy->cpus);
1036 static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
1038 struct kobject *kobj;
1039 struct completion *cmp;
1041 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1042 CPUFREQ_REMOVE_POLICY, policy);
1044 down_read(&policy->rwsem);
1045 kobj = &policy->kobj;
1046 cmp = &policy->kobj_unregister;
1047 up_read(&policy->rwsem);
1051 * We need to make sure that the underlying kobj is
1052 * actually not referenced anymore by anybody before we
1053 * proceed with unloading.
1055 pr_debug("waiting for dropping of refcount\n");
1056 wait_for_completion(cmp);
1057 pr_debug("wait complete\n");
1060 static void cpufreq_policy_free(struct cpufreq_policy *policy)
1062 free_cpumask_var(policy->related_cpus);
1063 free_cpumask_var(policy->cpus);
1067 static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
1069 if (WARN_ON(cpu == policy->cpu))
1072 down_write(&policy->rwsem);
1074 policy->last_cpu = policy->cpu;
1077 up_write(&policy->rwsem);
1079 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1080 CPUFREQ_UPDATE_POLICY_CPU, policy);
1083 static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1085 unsigned int j, cpu = dev->id;
1087 struct cpufreq_policy *policy;
1088 unsigned long flags;
1089 bool recover_policy = cpufreq_suspended;
1090 #ifdef CONFIG_HOTPLUG_CPU
1091 struct cpufreq_policy *tpolicy;
1094 if (cpu_is_offline(cpu))
1097 pr_debug("adding CPU %u\n", cpu);
1100 /* check whether a different CPU already registered this
1101 * CPU because it is in the same boat. */
1102 policy = cpufreq_cpu_get(cpu);
1103 if (unlikely(policy)) {
1104 cpufreq_cpu_put(policy);
1109 if (!down_read_trylock(&cpufreq_rwsem))
1112 #ifdef CONFIG_HOTPLUG_CPU
1113 /* Check if this cpu was hot-unplugged earlier and has siblings */
1114 read_lock_irqsave(&cpufreq_driver_lock, flags);
1115 list_for_each_entry(tpolicy, &cpufreq_policy_list, policy_list) {
1116 if (cpumask_test_cpu(cpu, tpolicy->related_cpus)) {
1117 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1118 ret = cpufreq_add_policy_cpu(tpolicy, cpu, dev);
1119 up_read(&cpufreq_rwsem);
1123 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1127 * Restore the saved policy when doing light-weight init and fall back
1128 * to the full init if that fails.
1130 policy = recover_policy ? cpufreq_policy_restore(cpu) : NULL;
1132 recover_policy = false;
1133 policy = cpufreq_policy_alloc();
1139 * In the resume path, since we restore a saved policy, the assignment
1140 * to policy->cpu is like an update of the existing policy, rather than
1141 * the creation of a brand new one. So we need to perform this update
1142 * by invoking update_policy_cpu().
1144 if (recover_policy && cpu != policy->cpu)
1145 update_policy_cpu(policy, cpu);
1149 cpumask_copy(policy->cpus, cpumask_of(cpu));
1151 init_completion(&policy->kobj_unregister);
1152 INIT_WORK(&policy->update, handle_update);
1154 /* call driver. From then on the cpufreq must be able
1155 * to accept all calls to ->verify and ->setpolicy for this CPU
1157 ret = cpufreq_driver->init(policy);
1159 pr_debug("initialization failed\n");
1160 goto err_set_policy_cpu;
1163 /* related cpus should atleast have policy->cpus */
1164 cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus);
1167 * affected cpus must always be the one, which are online. We aren't
1168 * managing offline cpus here.
1170 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1172 if (!recover_policy) {
1173 policy->user_policy.min = policy->min;
1174 policy->user_policy.max = policy->max;
1177 down_write(&policy->rwsem);
1178 write_lock_irqsave(&cpufreq_driver_lock, flags);
1179 for_each_cpu(j, policy->cpus)
1180 per_cpu(cpufreq_cpu_data, j) = policy;
1181 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1183 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
1184 policy->cur = cpufreq_driver->get(policy->cpu);
1186 pr_err("%s: ->get() failed\n", __func__);
1192 * Sometimes boot loaders set CPU frequency to a value outside of
1193 * frequency table present with cpufreq core. In such cases CPU might be
1194 * unstable if it has to run on that frequency for long duration of time
1195 * and so its better to set it to a frequency which is specified in
1196 * freq-table. This also makes cpufreq stats inconsistent as
1197 * cpufreq-stats would fail to register because current frequency of CPU
1198 * isn't found in freq-table.
1200 * Because we don't want this change to effect boot process badly, we go
1201 * for the next freq which is >= policy->cur ('cur' must be set by now,
1202 * otherwise we will end up setting freq to lowest of the table as 'cur'
1203 * is initialized to zero).
1205 * We are passing target-freq as "policy->cur - 1" otherwise
1206 * __cpufreq_driver_target() would simply fail, as policy->cur will be
1207 * equal to target-freq.
1209 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1211 /* Are we running at unknown frequency ? */
1212 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1213 if (ret == -EINVAL) {
1214 /* Warn user and fix it */
1215 pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1216 __func__, policy->cpu, policy->cur);
1217 ret = __cpufreq_driver_target(policy, policy->cur - 1,
1218 CPUFREQ_RELATION_L);
1221 * Reaching here after boot in a few seconds may not
1222 * mean that system will remain stable at "unknown"
1223 * frequency for longer duration. Hence, a BUG_ON().
1226 pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1227 __func__, policy->cpu, policy->cur);
1231 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1232 CPUFREQ_START, policy);
1234 if (!recover_policy) {
1235 ret = cpufreq_add_dev_interface(policy, dev);
1237 goto err_out_unregister;
1238 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1239 CPUFREQ_CREATE_POLICY, policy);
1242 write_lock_irqsave(&cpufreq_driver_lock, flags);
1243 list_add(&policy->policy_list, &cpufreq_policy_list);
1244 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1246 cpufreq_init_policy(policy);
1248 if (!recover_policy) {
1249 policy->user_policy.policy = policy->policy;
1250 policy->user_policy.governor = policy->governor;
1252 up_write(&policy->rwsem);
1254 kobject_uevent(&policy->kobj, KOBJ_ADD);
1255 up_read(&cpufreq_rwsem);
1257 pr_debug("initialization complete\n");
1263 write_lock_irqsave(&cpufreq_driver_lock, flags);
1264 for_each_cpu(j, policy->cpus)
1265 per_cpu(cpufreq_cpu_data, j) = NULL;
1266 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1268 if (cpufreq_driver->exit)
1269 cpufreq_driver->exit(policy);
1271 if (recover_policy) {
1272 /* Do not leave stale fallback data behind. */
1273 per_cpu(cpufreq_cpu_data_fallback, cpu) = NULL;
1274 cpufreq_policy_put_kobj(policy);
1276 cpufreq_policy_free(policy);
1279 up_read(&cpufreq_rwsem);
1285 * cpufreq_add_dev - add a CPU device
1287 * Adds the cpufreq interface for a CPU device.
1289 * The Oracle says: try running cpufreq registration/unregistration concurrently
1290 * with with cpu hotplugging and all hell will break loose. Tried to clean this
1291 * mess up, but more thorough testing is needed. - Mathieu
1293 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1295 return __cpufreq_add_dev(dev, sif);
1298 static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy,
1299 unsigned int old_cpu)
1301 struct device *cpu_dev;
1304 /* first sibling now owns the new sysfs dir */
1305 cpu_dev = get_cpu_device(cpumask_any_but(policy->cpus, old_cpu));
1307 sysfs_remove_link(&cpu_dev->kobj, "cpufreq");
1308 ret = kobject_move(&policy->kobj, &cpu_dev->kobj);
1310 pr_err("%s: Failed to move kobj: %d\n", __func__, ret);
1312 down_write(&policy->rwsem);
1313 cpumask_set_cpu(old_cpu, policy->cpus);
1314 up_write(&policy->rwsem);
1316 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
1325 static int __cpufreq_remove_dev_prepare(struct device *dev,
1326 struct subsys_interface *sif)
1328 unsigned int cpu = dev->id, cpus;
1330 unsigned long flags;
1331 struct cpufreq_policy *policy;
1333 pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
1335 write_lock_irqsave(&cpufreq_driver_lock, flags);
1337 policy = per_cpu(cpufreq_cpu_data, cpu);
1339 /* Save the policy somewhere when doing a light-weight tear-down */
1340 if (cpufreq_suspended)
1341 per_cpu(cpufreq_cpu_data_fallback, cpu) = policy;
1343 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1346 pr_debug("%s: No cpu_data found\n", __func__);
1351 ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
1353 pr_err("%s: Failed to stop governor\n", __func__);
1358 if (!cpufreq_driver->setpolicy)
1359 strncpy(per_cpu(cpufreq_cpu_governor, cpu),
1360 policy->governor->name, CPUFREQ_NAME_LEN);
1362 down_read(&policy->rwsem);
1363 cpus = cpumask_weight(policy->cpus);
1364 up_read(&policy->rwsem);
1366 if (cpu != policy->cpu) {
1367 sysfs_remove_link(&dev->kobj, "cpufreq");
1368 } else if (cpus > 1) {
1369 new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu);
1371 update_policy_cpu(policy, new_cpu);
1373 if (!cpufreq_suspended)
1374 pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n",
1375 __func__, new_cpu, cpu);
1377 } else if (cpufreq_driver->stop_cpu && cpufreq_driver->setpolicy) {
1378 cpufreq_driver->stop_cpu(policy);
1384 static int __cpufreq_remove_dev_finish(struct device *dev,
1385 struct subsys_interface *sif)
1387 unsigned int cpu = dev->id, cpus;
1389 unsigned long flags;
1390 struct cpufreq_policy *policy;
1392 read_lock_irqsave(&cpufreq_driver_lock, flags);
1393 policy = per_cpu(cpufreq_cpu_data, cpu);
1394 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1397 pr_debug("%s: No cpu_data found\n", __func__);
1401 down_write(&policy->rwsem);
1402 cpus = cpumask_weight(policy->cpus);
1405 cpumask_clear_cpu(cpu, policy->cpus);
1406 up_write(&policy->rwsem);
1408 /* If cpu is last user of policy, free policy */
1411 ret = __cpufreq_governor(policy,
1412 CPUFREQ_GOV_POLICY_EXIT);
1414 pr_err("%s: Failed to exit governor\n",
1420 if (!cpufreq_suspended)
1421 cpufreq_policy_put_kobj(policy);
1424 * Perform the ->exit() even during light-weight tear-down,
1425 * since this is a core component, and is essential for the
1426 * subsequent light-weight ->init() to succeed.
1428 if (cpufreq_driver->exit)
1429 cpufreq_driver->exit(policy);
1431 /* Remove policy from list of active policies */
1432 write_lock_irqsave(&cpufreq_driver_lock, flags);
1433 list_del(&policy->policy_list);
1434 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1436 if (!cpufreq_suspended)
1437 cpufreq_policy_free(policy);
1438 } else if (has_target()) {
1439 ret = __cpufreq_governor(policy, CPUFREQ_GOV_START);
1441 ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
1444 pr_err("%s: Failed to start governor\n", __func__);
1449 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1454 * cpufreq_remove_dev - remove a CPU device
1456 * Removes the cpufreq interface for a CPU device.
1458 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1460 unsigned int cpu = dev->id;
1463 if (cpu_is_offline(cpu))
1466 ret = __cpufreq_remove_dev_prepare(dev, sif);
1469 ret = __cpufreq_remove_dev_finish(dev, sif);
1474 static void handle_update(struct work_struct *work)
1476 struct cpufreq_policy *policy =
1477 container_of(work, struct cpufreq_policy, update);
1478 unsigned int cpu = policy->cpu;
1479 pr_debug("handle_update for cpu %u called\n", cpu);
1480 cpufreq_update_policy(cpu);
1484 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1487 * @old_freq: CPU frequency the kernel thinks the CPU runs at
1488 * @new_freq: CPU frequency the CPU actually runs at
1490 * We adjust to current frequency first, and need to clean up later.
1491 * So either call to cpufreq_update_policy() or schedule handle_update()).
1493 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1494 unsigned int new_freq)
1496 struct cpufreq_policy *policy;
1497 struct cpufreq_freqs freqs;
1498 unsigned long flags;
1500 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
1501 old_freq, new_freq);
1503 freqs.old = old_freq;
1504 freqs.new = new_freq;
1506 read_lock_irqsave(&cpufreq_driver_lock, flags);
1507 policy = per_cpu(cpufreq_cpu_data, cpu);
1508 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1510 cpufreq_notify_transition(policy, &freqs, CPUFREQ_PRECHANGE);
1511 cpufreq_notify_transition(policy, &freqs, CPUFREQ_POSTCHANGE);
1515 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1518 * This is the last known freq, without actually getting it from the driver.
1519 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1521 unsigned int cpufreq_quick_get(unsigned int cpu)
1523 struct cpufreq_policy *policy;
1524 unsigned int ret_freq = 0;
1526 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get)
1527 return cpufreq_driver->get(cpu);
1529 policy = cpufreq_cpu_get(cpu);
1531 ret_freq = policy->cur;
1532 cpufreq_cpu_put(policy);
1537 EXPORT_SYMBOL(cpufreq_quick_get);
1540 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1543 * Just return the max possible frequency for a given CPU.
1545 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1547 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1548 unsigned int ret_freq = 0;
1551 ret_freq = policy->max;
1552 cpufreq_cpu_put(policy);
1557 EXPORT_SYMBOL(cpufreq_quick_get_max);
1559 static unsigned int __cpufreq_get(unsigned int cpu)
1561 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1562 unsigned int ret_freq = 0;
1564 if (!cpufreq_driver->get)
1567 ret_freq = cpufreq_driver->get(cpu);
1569 if (ret_freq && policy->cur &&
1570 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1571 /* verify no discrepancy between actual and
1572 saved value exists */
1573 if (unlikely(ret_freq != policy->cur)) {
1574 cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1575 schedule_work(&policy->update);
1583 * cpufreq_get - get the current CPU frequency (in kHz)
1586 * Get the CPU current (static) CPU frequency
1588 unsigned int cpufreq_get(unsigned int cpu)
1590 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1591 unsigned int ret_freq = 0;
1594 down_read(&policy->rwsem);
1595 ret_freq = __cpufreq_get(cpu);
1596 up_read(&policy->rwsem);
1598 cpufreq_cpu_put(policy);
1603 EXPORT_SYMBOL(cpufreq_get);
1605 static struct subsys_interface cpufreq_interface = {
1607 .subsys = &cpu_subsys,
1608 .add_dev = cpufreq_add_dev,
1609 .remove_dev = cpufreq_remove_dev,
1613 * In case platform wants some specific frequency to be configured
1616 int cpufreq_generic_suspend(struct cpufreq_policy *policy)
1620 if (!policy->suspend_freq) {
1621 pr_err("%s: suspend_freq can't be zero\n", __func__);
1625 pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1626 policy->suspend_freq);
1628 ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1629 CPUFREQ_RELATION_H);
1631 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1632 __func__, policy->suspend_freq, ret);
1636 EXPORT_SYMBOL(cpufreq_generic_suspend);
1639 * cpufreq_suspend() - Suspend CPUFreq governors
1641 * Called during system wide Suspend/Hibernate cycles for suspending governors
1642 * as some platforms can't change frequency after this point in suspend cycle.
1643 * Because some of the devices (like: i2c, regulators, etc) they use for
1644 * changing frequency are suspended quickly after this point.
1646 void cpufreq_suspend(void)
1648 struct cpufreq_policy *policy;
1650 if (!cpufreq_driver)
1656 pr_debug("%s: Suspending Governors\n", __func__);
1658 list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1659 if (__cpufreq_governor(policy, CPUFREQ_GOV_STOP))
1660 pr_err("%s: Failed to stop governor for policy: %p\n",
1662 else if (cpufreq_driver->suspend
1663 && cpufreq_driver->suspend(policy))
1664 pr_err("%s: Failed to suspend driver: %p\n", __func__,
1668 cpufreq_suspended = true;
1672 * cpufreq_resume() - Resume CPUFreq governors
1674 * Called during system wide Suspend/Hibernate cycle for resuming governors that
1675 * are suspended with cpufreq_suspend().
1677 void cpufreq_resume(void)
1679 struct cpufreq_policy *policy;
1681 if (!cpufreq_driver)
1687 pr_debug("%s: Resuming Governors\n", __func__);
1689 cpufreq_suspended = false;
1691 list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
1692 if (cpufreq_driver->resume && cpufreq_driver->resume(policy))
1693 pr_err("%s: Failed to resume driver: %p\n", __func__,
1695 else if (__cpufreq_governor(policy, CPUFREQ_GOV_START)
1696 || __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))
1697 pr_err("%s: Failed to start governor for policy: %p\n",
1701 * schedule call cpufreq_update_policy() for boot CPU, i.e. last
1702 * policy in list. It will verify that the current freq is in
1703 * sync with what we believe it to be.
1705 if (list_is_last(&policy->policy_list, &cpufreq_policy_list))
1706 schedule_work(&policy->update);
1711 * cpufreq_get_current_driver - return current driver's name
1713 * Return the name string of the currently loaded cpufreq driver
1716 const char *cpufreq_get_current_driver(void)
1719 return cpufreq_driver->name;
1723 EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
1725 /*********************************************************************
1726 * NOTIFIER LISTS INTERFACE *
1727 *********************************************************************/
1730 * cpufreq_register_notifier - register a driver with cpufreq
1731 * @nb: notifier function to register
1732 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1734 * Add a driver to one of two lists: either a list of drivers that
1735 * are notified about clock rate changes (once before and once after
1736 * the transition), or a list of drivers that are notified about
1737 * changes in cpufreq policy.
1739 * This function may sleep, and has the same return conditions as
1740 * blocking_notifier_chain_register.
1742 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1746 if (cpufreq_disabled())
1749 WARN_ON(!init_cpufreq_transition_notifier_list_called);
1752 case CPUFREQ_TRANSITION_NOTIFIER:
1753 ret = srcu_notifier_chain_register(
1754 &cpufreq_transition_notifier_list, nb);
1756 case CPUFREQ_POLICY_NOTIFIER:
1757 ret = blocking_notifier_chain_register(
1758 &cpufreq_policy_notifier_list, nb);
1766 EXPORT_SYMBOL(cpufreq_register_notifier);
1769 * cpufreq_unregister_notifier - unregister a driver with cpufreq
1770 * @nb: notifier block to be unregistered
1771 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1773 * Remove a driver from the CPU frequency notifier list.
1775 * This function may sleep, and has the same return conditions as
1776 * blocking_notifier_chain_unregister.
1778 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1782 if (cpufreq_disabled())
1786 case CPUFREQ_TRANSITION_NOTIFIER:
1787 ret = srcu_notifier_chain_unregister(
1788 &cpufreq_transition_notifier_list, nb);
1790 case CPUFREQ_POLICY_NOTIFIER:
1791 ret = blocking_notifier_chain_unregister(
1792 &cpufreq_policy_notifier_list, nb);
1800 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1803 /*********************************************************************
1805 *********************************************************************/
1807 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1808 unsigned int target_freq,
1809 unsigned int relation)
1811 int retval = -EINVAL;
1812 unsigned int old_target_freq = target_freq;
1814 if (cpufreq_disabled())
1817 /* Make sure that target_freq is within supported range */
1818 if (target_freq > policy->max)
1819 target_freq = policy->max;
1820 if (target_freq < policy->min)
1821 target_freq = policy->min;
1823 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1824 policy->cpu, target_freq, relation, old_target_freq);
1827 * This might look like a redundant call as we are checking it again
1828 * after finding index. But it is left intentionally for cases where
1829 * exactly same freq is called again and so we can save on few function
1832 if (target_freq == policy->cur)
1835 if (cpufreq_driver->target)
1836 retval = cpufreq_driver->target(policy, target_freq, relation);
1837 else if (cpufreq_driver->target_index) {
1838 struct cpufreq_frequency_table *freq_table;
1839 struct cpufreq_freqs freqs;
1843 freq_table = cpufreq_frequency_get_table(policy->cpu);
1844 if (unlikely(!freq_table)) {
1845 pr_err("%s: Unable to find freq_table\n", __func__);
1849 retval = cpufreq_frequency_table_target(policy, freq_table,
1850 target_freq, relation, &index);
1851 if (unlikely(retval)) {
1852 pr_err("%s: Unable to find matching freq\n", __func__);
1856 if (freq_table[index].frequency == policy->cur) {
1861 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
1864 freqs.old = policy->cur;
1865 freqs.new = freq_table[index].frequency;
1868 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
1869 __func__, policy->cpu, freqs.old, freqs.new);
1871 cpufreq_notify_transition(policy, &freqs,
1875 retval = cpufreq_driver->target_index(policy, index);
1877 pr_err("%s: Failed to change cpu frequency: %d\n",
1881 cpufreq_notify_post_transition(policy, &freqs, retval);
1887 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1889 int cpufreq_driver_target(struct cpufreq_policy *policy,
1890 unsigned int target_freq,
1891 unsigned int relation)
1895 down_write(&policy->rwsem);
1897 ret = __cpufreq_driver_target(policy, target_freq, relation);
1899 up_write(&policy->rwsem);
1903 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1906 * when "event" is CPUFREQ_GOV_LIMITS
1909 static int __cpufreq_governor(struct cpufreq_policy *policy,
1914 /* Only must be defined when default governor is known to have latency
1915 restrictions, like e.g. conservative or ondemand.
1916 That this is the case is already ensured in Kconfig
1918 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1919 struct cpufreq_governor *gov = &cpufreq_gov_performance;
1921 struct cpufreq_governor *gov = NULL;
1924 /* Don't start any governor operations if we are entering suspend */
1925 if (cpufreq_suspended)
1928 if (policy->governor->max_transition_latency &&
1929 policy->cpuinfo.transition_latency >
1930 policy->governor->max_transition_latency) {
1934 pr_warn("%s governor failed, too long transition latency of HW, fallback to %s governor\n",
1935 policy->governor->name, gov->name);
1936 policy->governor = gov;
1940 if (event == CPUFREQ_GOV_POLICY_INIT)
1941 if (!try_module_get(policy->governor->owner))
1944 pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1945 policy->cpu, event);
1947 mutex_lock(&cpufreq_governor_lock);
1948 if ((policy->governor_enabled && event == CPUFREQ_GOV_START)
1949 || (!policy->governor_enabled
1950 && (event == CPUFREQ_GOV_LIMITS || event == CPUFREQ_GOV_STOP))) {
1951 mutex_unlock(&cpufreq_governor_lock);
1955 if (event == CPUFREQ_GOV_STOP)
1956 policy->governor_enabled = false;
1957 else if (event == CPUFREQ_GOV_START)
1958 policy->governor_enabled = true;
1960 mutex_unlock(&cpufreq_governor_lock);
1962 ret = policy->governor->governor(policy, event);
1965 if (event == CPUFREQ_GOV_POLICY_INIT)
1966 policy->governor->initialized++;
1967 else if (event == CPUFREQ_GOV_POLICY_EXIT)
1968 policy->governor->initialized--;
1970 /* Restore original values */
1971 mutex_lock(&cpufreq_governor_lock);
1972 if (event == CPUFREQ_GOV_STOP)
1973 policy->governor_enabled = true;
1974 else if (event == CPUFREQ_GOV_START)
1975 policy->governor_enabled = false;
1976 mutex_unlock(&cpufreq_governor_lock);
1979 if (((event == CPUFREQ_GOV_POLICY_INIT) && ret) ||
1980 ((event == CPUFREQ_GOV_POLICY_EXIT) && !ret))
1981 module_put(policy->governor->owner);
1986 int cpufreq_register_governor(struct cpufreq_governor *governor)
1993 if (cpufreq_disabled())
1996 mutex_lock(&cpufreq_governor_mutex);
1998 governor->initialized = 0;
2000 if (__find_governor(governor->name) == NULL) {
2002 list_add(&governor->governor_list, &cpufreq_governor_list);
2005 mutex_unlock(&cpufreq_governor_mutex);
2008 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2010 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2017 if (cpufreq_disabled())
2020 for_each_present_cpu(cpu) {
2021 if (cpu_online(cpu))
2023 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
2024 strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
2027 mutex_lock(&cpufreq_governor_mutex);
2028 list_del(&governor->governor_list);
2029 mutex_unlock(&cpufreq_governor_mutex);
2032 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2035 /*********************************************************************
2036 * POLICY INTERFACE *
2037 *********************************************************************/
2040 * cpufreq_get_policy - get the current cpufreq_policy
2041 * @policy: struct cpufreq_policy into which the current cpufreq_policy
2044 * Reads the current cpufreq policy.
2046 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2048 struct cpufreq_policy *cpu_policy;
2052 cpu_policy = cpufreq_cpu_get(cpu);
2056 memcpy(policy, cpu_policy, sizeof(*policy));
2058 cpufreq_cpu_put(cpu_policy);
2061 EXPORT_SYMBOL(cpufreq_get_policy);
2064 * policy : current policy.
2065 * new_policy: policy to be set.
2067 static int cpufreq_set_policy(struct cpufreq_policy *policy,
2068 struct cpufreq_policy *new_policy)
2070 struct cpufreq_governor *old_gov;
2073 pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2074 new_policy->cpu, new_policy->min, new_policy->max);
2076 memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
2078 if (new_policy->min > policy->max || new_policy->max < policy->min)
2081 /* verify the cpu speed can be set within this limit */
2082 ret = cpufreq_driver->verify(new_policy);
2086 /* adjust if necessary - all reasons */
2087 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2088 CPUFREQ_ADJUST, new_policy);
2090 /* adjust if necessary - hardware incompatibility*/
2091 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2092 CPUFREQ_INCOMPATIBLE, new_policy);
2095 * verify the cpu speed can be set within this limit, which might be
2096 * different to the first one
2098 ret = cpufreq_driver->verify(new_policy);
2102 /* notification of the new policy */
2103 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
2104 CPUFREQ_NOTIFY, new_policy);
2106 policy->min = new_policy->min;
2107 policy->max = new_policy->max;
2109 pr_debug("new min and max freqs are %u - %u kHz\n",
2110 policy->min, policy->max);
2112 if (cpufreq_driver->setpolicy) {
2113 policy->policy = new_policy->policy;
2114 pr_debug("setting range\n");
2115 return cpufreq_driver->setpolicy(new_policy);
2118 if (new_policy->governor == policy->governor)
2121 pr_debug("governor switch\n");
2123 /* save old, working values */
2124 old_gov = policy->governor;
2125 /* end old governor */
2127 __cpufreq_governor(policy, CPUFREQ_GOV_STOP);
2128 up_write(&policy->rwsem);
2129 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2130 down_write(&policy->rwsem);
2133 /* start new governor */
2134 policy->governor = new_policy->governor;
2135 if (!__cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT)) {
2136 if (!__cpufreq_governor(policy, CPUFREQ_GOV_START))
2139 up_write(&policy->rwsem);
2140 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT);
2141 down_write(&policy->rwsem);
2144 /* new governor failed, so re-start old one */
2145 pr_debug("starting governor %s failed\n", policy->governor->name);
2147 policy->governor = old_gov;
2148 __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_INIT);
2149 __cpufreq_governor(policy, CPUFREQ_GOV_START);
2155 pr_debug("governor: change or update limits\n");
2156 return __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2160 * cpufreq_update_policy - re-evaluate an existing cpufreq policy
2161 * @cpu: CPU which shall be re-evaluated
2163 * Useful for policy notifiers which have different necessities
2164 * at different times.
2166 int cpufreq_update_policy(unsigned int cpu)
2168 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
2169 struct cpufreq_policy new_policy;
2177 down_write(&policy->rwsem);
2179 pr_debug("updating policy for CPU %u\n", cpu);
2180 memcpy(&new_policy, policy, sizeof(*policy));
2181 new_policy.min = policy->user_policy.min;
2182 new_policy.max = policy->user_policy.max;
2183 new_policy.policy = policy->user_policy.policy;
2184 new_policy.governor = policy->user_policy.governor;
2187 * BIOS might change freq behind our back
2188 * -> ask driver for current freq and notify governors about a change
2190 if (cpufreq_driver->get && !cpufreq_driver->setpolicy) {
2191 new_policy.cur = cpufreq_driver->get(cpu);
2192 if (WARN_ON(!new_policy.cur)) {
2198 pr_debug("Driver did not initialize current freq\n");
2199 policy->cur = new_policy.cur;
2201 if (policy->cur != new_policy.cur && has_target())
2202 cpufreq_out_of_sync(cpu, policy->cur,
2207 ret = cpufreq_set_policy(policy, &new_policy);
2209 up_write(&policy->rwsem);
2211 cpufreq_cpu_put(policy);
2215 EXPORT_SYMBOL(cpufreq_update_policy);
2217 static int cpufreq_cpu_callback(struct notifier_block *nfb,
2218 unsigned long action, void *hcpu)
2220 unsigned int cpu = (unsigned long)hcpu;
2223 dev = get_cpu_device(cpu);
2225 switch (action & ~CPU_TASKS_FROZEN) {
2227 __cpufreq_add_dev(dev, NULL);
2230 case CPU_DOWN_PREPARE:
2231 __cpufreq_remove_dev_prepare(dev, NULL);
2235 __cpufreq_remove_dev_finish(dev, NULL);
2238 case CPU_DOWN_FAILED:
2239 __cpufreq_add_dev(dev, NULL);
2246 static struct notifier_block __refdata cpufreq_cpu_notifier = {
2247 .notifier_call = cpufreq_cpu_callback,
2250 /*********************************************************************
2252 *********************************************************************/
2253 static int cpufreq_boost_set_sw(int state)
2255 struct cpufreq_frequency_table *freq_table;
2256 struct cpufreq_policy *policy;
2259 list_for_each_entry(policy, &cpufreq_policy_list, policy_list) {
2260 freq_table = cpufreq_frequency_get_table(policy->cpu);
2262 ret = cpufreq_frequency_table_cpuinfo(policy,
2265 pr_err("%s: Policy frequency update failed\n",
2269 policy->user_policy.max = policy->max;
2270 __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS);
2277 int cpufreq_boost_trigger_state(int state)
2279 unsigned long flags;
2282 if (cpufreq_driver->boost_enabled == state)
2285 write_lock_irqsave(&cpufreq_driver_lock, flags);
2286 cpufreq_driver->boost_enabled = state;
2287 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2289 ret = cpufreq_driver->set_boost(state);
2291 write_lock_irqsave(&cpufreq_driver_lock, flags);
2292 cpufreq_driver->boost_enabled = !state;
2293 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2295 pr_err("%s: Cannot %s BOOST\n",
2296 __func__, state ? "enable" : "disable");
2302 int cpufreq_boost_supported(void)
2304 if (likely(cpufreq_driver))
2305 return cpufreq_driver->boost_supported;
2309 EXPORT_SYMBOL_GPL(cpufreq_boost_supported);
2311 int cpufreq_boost_enabled(void)
2313 return cpufreq_driver->boost_enabled;
2315 EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2317 /*********************************************************************
2318 * REGISTER / UNREGISTER CPUFREQ DRIVER *
2319 *********************************************************************/
2322 * cpufreq_register_driver - register a CPU Frequency driver
2323 * @driver_data: A struct cpufreq_driver containing the values#
2324 * submitted by the CPU Frequency driver.
2326 * Registers a CPU Frequency driver to this core code. This code
2327 * returns zero on success, -EBUSY when another driver got here first
2328 * (and isn't unregistered in the meantime).
2331 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
2333 unsigned long flags;
2336 if (cpufreq_disabled())
2339 if (!driver_data || !driver_data->verify || !driver_data->init ||
2340 !(driver_data->setpolicy || driver_data->target_index ||
2341 driver_data->target) ||
2342 (driver_data->setpolicy && (driver_data->target_index ||
2343 driver_data->target)))
2346 pr_debug("trying to register driver %s\n", driver_data->name);
2348 if (driver_data->setpolicy)
2349 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2351 write_lock_irqsave(&cpufreq_driver_lock, flags);
2352 if (cpufreq_driver) {
2353 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2356 cpufreq_driver = driver_data;
2357 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2359 if (cpufreq_boost_supported()) {
2361 * Check if driver provides function to enable boost -
2362 * if not, use cpufreq_boost_set_sw as default
2364 if (!cpufreq_driver->set_boost)
2365 cpufreq_driver->set_boost = cpufreq_boost_set_sw;
2367 ret = cpufreq_sysfs_create_file(&boost.attr);
2369 pr_err("%s: cannot register global BOOST sysfs file\n",
2371 goto err_null_driver;
2375 ret = subsys_interface_register(&cpufreq_interface);
2377 goto err_boost_unreg;
2379 if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
2383 /* check for at least one working CPU */
2384 for (i = 0; i < nr_cpu_ids; i++)
2385 if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
2390 /* if all ->init() calls failed, unregister */
2392 pr_debug("no CPU initialized for driver %s\n",
2398 register_hotcpu_notifier(&cpufreq_cpu_notifier);
2399 pr_debug("driver %s up and running\n", driver_data->name);
2403 subsys_interface_unregister(&cpufreq_interface);
2405 if (cpufreq_boost_supported())
2406 cpufreq_sysfs_remove_file(&boost.attr);
2408 write_lock_irqsave(&cpufreq_driver_lock, flags);
2409 cpufreq_driver = NULL;
2410 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2413 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2416 * cpufreq_unregister_driver - unregister the current CPUFreq driver
2418 * Unregister the current CPUFreq driver. Only call this if you have
2419 * the right to do so, i.e. if you have succeeded in initialising before!
2420 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2421 * currently not initialised.
2423 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
2425 unsigned long flags;
2427 if (!cpufreq_driver || (driver != cpufreq_driver))
2430 pr_debug("unregistering driver %s\n", driver->name);
2432 subsys_interface_unregister(&cpufreq_interface);
2433 if (cpufreq_boost_supported())
2434 cpufreq_sysfs_remove_file(&boost.attr);
2436 unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
2438 down_write(&cpufreq_rwsem);
2439 write_lock_irqsave(&cpufreq_driver_lock, flags);
2441 cpufreq_driver = NULL;
2443 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2444 up_write(&cpufreq_rwsem);
2448 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
2450 static int __init cpufreq_core_init(void)
2452 if (cpufreq_disabled())
2455 cpufreq_global_kobject = kobject_create();
2456 BUG_ON(!cpufreq_global_kobject);
2460 core_initcall(cpufreq_core_init);