drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #include <linux/kernel.h>
  19 #include <linux/module.h>
  20 #include <linux/init.h>
  21 #include <linux/notifier.h>
  22 #include <linux/cpufreq.h>
  23 #include <linux/delay.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/spinlock.h>
  26 #include <linux/device.h>
  27 #include <linux/slab.h>
  28 #include <linux/cpu.h>
  29 #include <linux/completion.h>
  30 #include <linux/mutex.h>
  31
  32 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, \
  33                                                 "cpufreq-core", msg)
  34
  35 /**
  36  * The "cpufreq driver" - the arch- or hardware-dependent low
  37  * level driver of CPUFreq support, and its spinlock. This lock
  38  * also protects the cpufreq_cpu_data array.
  39  */
  40 static struct cpufreq_driver *cpufreq_driver;
  41 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  42 #ifdef CONFIG_HOTPLUG_CPU
  43 /* This one keeps track of the previously set governor of a removed CPU */
  44 static DEFINE_PER_CPU(struct cpufreq_governor *, cpufreq_cpu_governor);
  45 #endif
  46 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  47
  48 /*
  49  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  50  * all cpufreq/hotplug/workqueue/etc related lock issues.
  51  *
  52  * The rules for this semaphore:
  53  * - Any routine that wants to read from the policy structure will
  54  *   do a down_read on this semaphore.
  55  * - Any routine that will write to the policy structure and/or may take away
  56  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  57  *   mode before doing so.
  58  *
  59  * Additional rules:
  60  * - All holders of the lock should check to make sure that the CPU they
  61  *   are concerned with are online after they get the lock.
  62  * - Governor routines that can be called in cpufreq hotplug path should not
  63  *   take this sem as top level hotplug notifier handler takes this.
  64  */
  65 static DEFINE_PER_CPU(int, policy_cpu);
  66 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  67
  68 #define lock_policy_rwsem(mode, cpu)                                    \
  69 int lock_policy_rwsem_##mode                                            \
  70 (int cpu)                                                               \
  71 {                                                                       \
  72         int policy_cpu = per_cpu(policy_cpu, cpu);                      \
  73         BUG_ON(policy_cpu == -1);                                       \
  74         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  75         if (unlikely(!cpu_online(cpu))) {                               \
  76                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  77                 return -1;                                              \
  78         }                                                               \
  79                                                                         \
  80         return 0;                                                       \
  81 }
  82
  83 lock_policy_rwsem(read, cpu);
  84 EXPORT_SYMBOL_GPL(lock_policy_rwsem_read);
  85
  86 lock_policy_rwsem(write, cpu);
  87 EXPORT_SYMBOL_GPL(lock_policy_rwsem_write);
  88
  89 void unlock_policy_rwsem_read(int cpu)
  90 {
  91         int policy_cpu = per_cpu(policy_cpu, cpu);
  92         BUG_ON(policy_cpu == -1);
  93         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  94 }
  95 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_read);
  96
  97 void unlock_policy_rwsem_write(int cpu)
  98 {
  99         int policy_cpu = per_cpu(policy_cpu, cpu);
 100         BUG_ON(policy_cpu == -1);
 101         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 102 }
 103 EXPORT_SYMBOL_GPL(unlock_policy_rwsem_write);
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy,
 108                 unsigned int event);
 109 static unsigned int __cpufreq_get(unsigned int cpu);
 110 static void handle_update(struct work_struct *work);
 111
 112 /**
 113  * Two notifier lists: the "policy" list is involved in the
 114  * validation process for a new CPU frequency policy; the
 115  * "transition" list for kernel code that needs to handle
 116  * changes to devices when the CPU clock speed changes.
 117  * The mutex locks both lists.
 118  */
 119 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 120 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 121
 122 static bool init_cpufreq_transition_notifier_list_called;
 123 static int __init init_cpufreq_transition_notifier_list(void)
 124 {
 125         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 126         init_cpufreq_transition_notifier_list_called = true;
 127         return 0;
 128 }
 129 pure_initcall(init_cpufreq_transition_notifier_list);
 130
 131 static LIST_HEAD(cpufreq_governor_list);
 132 static DEFINE_MUTEX(cpufreq_governor_mutex);
 133
 134 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 135 {
 136         struct cpufreq_policy *data;
 137         unsigned long flags;
 138
 139         if (cpu >= nr_cpu_ids)
 140                 goto err_out;
 141
 142         /* get the cpufreq driver */
 143         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 144
 145         if (!cpufreq_driver)
 146                 goto err_out_unlock;
 147
 148         if (!try_module_get(cpufreq_driver->owner))
 149                 goto err_out_unlock;
 150
 151
 152         /* get the CPU */
 153         data = per_cpu(cpufreq_cpu_data, cpu);
 154
 155         if (!data)
 156                 goto err_out_put_module;
 157
 158         if (!kobject_get(&data->kobj))
 159                 goto err_out_put_module;
 160
 161         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 162         return data;
 163
 164 err_out_put_module:
 165         module_put(cpufreq_driver->owner);
 166 err_out_unlock:
 167         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 168 err_out:
 169         return NULL;
 170 }
 171 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 172
 173
 174 void cpufreq_cpu_put(struct cpufreq_policy *data)
 175 {
 176         kobject_put(&data->kobj);
 177         module_put(cpufreq_driver->owner);
 178 }
 179 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 180
 181
 182 /*********************************************************************
 183  *                     UNIFIED DEBUG HELPERS                         *
 184  *********************************************************************/
 185 #ifdef CONFIG_CPU_FREQ_DEBUG
 186
 187 /* what part(s) of the CPUfreq subsystem are debugged? */
 188 static unsigned int debug;
 189
 190 /* is the debug output ratelimit'ed using printk_ratelimit? User can
 191  * set or modify this value.
 192  */
 193 static unsigned int debug_ratelimit = 1;
 194
 195 /* is the printk_ratelimit'ing enabled? It's enabled after a successful
 196  * loading of a cpufreq driver, temporarily disabled when a new policy
 197  * is set, and disabled upon cpufreq driver removal
 198  */
 199 static unsigned int disable_ratelimit = 1;
 200 static DEFINE_SPINLOCK(disable_ratelimit_lock);
 201
 202 static void cpufreq_debug_enable_ratelimit(void)
 203 {
 204         unsigned long flags;
 205
 206         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 207         if (disable_ratelimit)
 208                 disable_ratelimit--;
 209         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 210 }
 211
 212 static void cpufreq_debug_disable_ratelimit(void)
 213 {
 214         unsigned long flags;
 215
 216         spin_lock_irqsave(&disable_ratelimit_lock, flags);
 217         disable_ratelimit++;
 218         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 219 }
 220
 221 void cpufreq_debug_printk(unsigned int type, const char *prefix,
 222                         const char *fmt, ...)
 223 {
 224         char s[256];
 225         va_list args;
 226         unsigned int len;
 227         unsigned long flags;
 228
 229         WARN_ON(!prefix);
 230         if (type & debug) {
 231                 spin_lock_irqsave(&disable_ratelimit_lock, flags);
 232                 if (!disable_ratelimit && debug_ratelimit
 233                                         && !printk_ratelimit()) {
 234                         spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 235                         return;
 236                 }
 237                 spin_unlock_irqrestore(&disable_ratelimit_lock, flags);
 238
 239                 len = snprintf(s, 256, KERN_DEBUG "%s: ", prefix);
 240
 241                 va_start(args, fmt);
 242                 len += vsnprintf(&s[len], (256 - len), fmt, args);
 243                 va_end(args);
 244
 245                 printk(s);
 246
 247                 WARN_ON(len < 5);
 248         }
 249 }
 250 EXPORT_SYMBOL(cpufreq_debug_printk);
 251
 252
 253 module_param(debug, uint, 0644);
 254 MODULE_PARM_DESC(debug, "CPUfreq debugging: add 1 to debug core,"
 255                         " 2 to debug drivers, and 4 to debug governors.");
 256
 257 module_param(debug_ratelimit, uint, 0644);
 258 MODULE_PARM_DESC(debug_ratelimit, "CPUfreq debugging:"
 259                                         " set to 0 to disable ratelimiting.");
 260
 261 #else /* !CONFIG_CPU_FREQ_DEBUG */
 262
 263 static inline void cpufreq_debug_enable_ratelimit(void) { return; }
 264 static inline void cpufreq_debug_disable_ratelimit(void) { return; }
 265
 266 #endif /* CONFIG_CPU_FREQ_DEBUG */
 267
 268
 269 /*********************************************************************
 270  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 271  *********************************************************************/
 272
 273 /**
 274  * adjust_jiffies - adjust the system "loops_per_jiffy"
 275  *
 276  * This function alters the system "loops_per_jiffy" for the clock
 277  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 278  * systems as each CPU might be scaled differently. So, use the arch
 279  * per-CPU loops_per_jiffy value wherever possible.
 280  */
 281 #ifndef CONFIG_SMP
 282 static unsigned long l_p_j_ref;
 283 static unsigned int  l_p_j_ref_freq;
 284
 285 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 286 {
 287         if (ci->flags & CPUFREQ_CONST_LOOPS)
 288                 return;
 289
 290         if (!l_p_j_ref_freq) {
 291                 l_p_j_ref = loops_per_jiffy;
 292                 l_p_j_ref_freq = ci->old;
 293                 dprintk("saving %lu as reference value for loops_per_jiffy; "
 294                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 295         }
 296         if ((val == CPUFREQ_PRECHANGE  && ci->old < ci->new) ||
 297             (val == CPUFREQ_POSTCHANGE && ci->old > ci->new) ||
 298             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 299                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 300                                                                 ci->new);
 301                 dprintk("scaling loops_per_jiffy to %lu "
 302                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 303         }
 304 }
 305 #else
 306 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 307 {
 308         return;
 309 }
 310 #endif
 311
 312
 313 /**
 314  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 315  * on frequency transition.
 316  *
 317  * This function calls the transition notifiers and the "adjust_jiffies"
 318  * function. It is called twice on all CPU frequency changes that have
 319  * external effects.
 320  */
 321 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 322 {
 323         struct cpufreq_policy *policy;
 324
 325         BUG_ON(irqs_disabled());
 326
 327         freqs->flags = cpufreq_driver->flags;
 328         dprintk("notification %u of frequency transition to %u kHz\n",
 329                 state, freqs->new);
 330
 331         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 332         switch (state) {
 333
 334         case CPUFREQ_PRECHANGE:
 335                 /* detect if the driver reported a value as "old frequency"
 336                  * which is not equal to what the cpufreq core thinks is
 337                  * "old frequency".
 338                  */
 339                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 340                         if ((policy) && (policy->cpu == freqs->cpu) &&
 341                             (policy->cur) && (policy->cur != freqs->old)) {
 342                                 dprintk("Warning: CPU frequency is"
 343                                         " %u, cpufreq assumed %u kHz.\n",
 344                                         freqs->old, policy->cur);
 345                                 freqs->old = policy->cur;
 346                         }
 347                 }
 348                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 349                                 CPUFREQ_PRECHANGE, freqs);
 350                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 351                 break;
 352
 353         case CPUFREQ_POSTCHANGE:
 354                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 355                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 356                                 CPUFREQ_POSTCHANGE, freqs);
 357                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 358                         policy->cur = freqs->new;
 359                 break;
 360         }
 361 }
 362 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 363
 364
 365
 366 /*********************************************************************
 367  *                          SYSFS INTERFACE                          *
 368  *********************************************************************/
 369
 370 static struct cpufreq_governor *__find_governor(const char *str_governor)
 371 {
 372         struct cpufreq_governor *t;
 373
 374         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 375                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 376                         return t;
 377
 378         return NULL;
 379 }
 380
 381 /**
 382  * cpufreq_parse_governor - parse a governor string
 383  */
 384 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 385                                 struct cpufreq_governor **governor)
 386 {
 387         int err = -EINVAL;
 388
 389         if (!cpufreq_driver)
 390                 goto out;
 391
 392         if (cpufreq_driver->setpolicy) {
 393                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 394                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 395                         err = 0;
 396                 } else if (!strnicmp(str_governor, "powersave",
 397                                                 CPUFREQ_NAME_LEN)) {
 398                         *policy = CPUFREQ_POLICY_POWERSAVE;
 399                         err = 0;
 400                 }
 401         } else if (cpufreq_driver->target) {
 402                 struct cpufreq_governor *t;
 403
 404                 mutex_lock(&cpufreq_governor_mutex);
 405
 406                 t = __find_governor(str_governor);
 407
 408                 if (t == NULL) {
 409                         char *name = kasprintf(GFP_KERNEL, "cpufreq_%s",
 410                                                                 str_governor);
 411
 412                         if (name) {
 413                                 int ret;
 414
 415                                 mutex_unlock(&cpufreq_governor_mutex);
 416                                 ret = request_module("%s", name);
 417                                 mutex_lock(&cpufreq_governor_mutex);
 418
 419                                 if (ret == 0)
 420                                         t = __find_governor(str_governor);
 421                         }
 422
 423                         kfree(name);
 424                 }
 425
 426                 if (t != NULL) {
 427                         *governor = t;
 428                         err = 0;
 429                 }
 430
 431                 mutex_unlock(&cpufreq_governor_mutex);
 432         }
 433 out:
 434         return err;
 435 }
 436
 437
 438 /**
 439  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 440  * print out cpufreq information
 441  *
 442  * Write out information from cpufreq_driver->policy[cpu]; object must be
 443  * "unsigned int".
 444  */
 445
 446 #define show_one(file_name, object)                     \
 447 static ssize_t show_##file_name                         \
 448 (struct cpufreq_policy *policy, char *buf)              \
 449 {                                                       \
 450         return sprintf(buf, "%u\n", policy->object);    \
 451 }
 452
 453 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 454 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 455 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 456 show_one(scaling_min_freq, min);
 457 show_one(scaling_max_freq, max);
 458 show_one(scaling_cur_freq, cur);
 459
 460 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 461                                 struct cpufreq_policy *policy);
 462
 463 /**
 464  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 465  */
 466 #define store_one(file_name, object)                    \
 467 static ssize_t store_##file_name                                        \
 468 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 469 {                                                                       \
 470         unsigned int ret = -EINVAL;                                     \
 471         struct cpufreq_policy new_policy;                               \
 472                                                                         \
 473         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 474         if (ret)                                                        \
 475                 return -EINVAL;                                         \
 476                                                                         \
 477         ret = sscanf(buf, "%u", &new_policy.object);                    \
 478         if (ret != 1)                                                   \
 479                 return -EINVAL;                                         \
 480                                                                         \
 481         ret = __cpufreq_set_policy(policy, &new_policy);                \
 482         policy->user_policy.object = policy->object;                    \
 483                                                                         \
 484         return ret ? ret : count;                                       \
 485 }
 486
 487 store_one(scaling_min_freq, min);
 488 store_one(scaling_max_freq, max);
 489
 490 /**
 491  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 492  */
 493 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 494                                         char *buf)
 495 {
 496         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 497         if (!cur_freq)
 498                 return sprintf(buf, "<unknown>");
 499         return sprintf(buf, "%u\n", cur_freq);
 500 }
 501
 502
 503 /**
 504  * show_scaling_governor - show the current policy for the specified CPU
 505  */
 506 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 507 {
 508         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 509                 return sprintf(buf, "powersave\n");
 510         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 511                 return sprintf(buf, "performance\n");
 512         else if (policy->governor)
 513                 return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n",
 514                                 policy->governor->name);
 515         return -EINVAL;
 516 }
 517
 518
 519 /**
 520  * store_scaling_governor - store policy for the specified CPU
 521  */
 522 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 523                                         const char *buf, size_t count)
 524 {
 525         unsigned int ret = -EINVAL;
 526         char    str_governor[16];
 527         struct cpufreq_policy new_policy;
 528
 529         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 530         if (ret)
 531                 return ret;
 532
 533         ret = sscanf(buf, "%15s", str_governor);
 534         if (ret != 1)
 535                 return -EINVAL;
 536
 537         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 538                                                 &new_policy.governor))
 539                 return -EINVAL;
 540
 541         /* Do not use cpufreq_set_policy here or the user_policy.max
 542            will be wrongly overridden */
 543         ret = __cpufreq_set_policy(policy, &new_policy);
 544
 545         policy->user_policy.policy = policy->policy;
 546         policy->user_policy.governor = policy->governor;
 547
 548         if (ret)
 549                 return ret;
 550         else
 551                 return count;
 552 }
 553
 554 /**
 555  * show_scaling_driver - show the cpufreq driver currently loaded
 556  */
 557 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 558 {
 559         return scnprintf(buf, CPUFREQ_NAME_LEN, "%s\n", cpufreq_driver->name);
 560 }
 561
 562 /**
 563  * show_scaling_available_governors - show the available CPUfreq governors
 564  */
 565 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 566                                                 char *buf)
 567 {
 568         ssize_t i = 0;
 569         struct cpufreq_governor *t;
 570
 571         if (!cpufreq_driver->target) {
 572                 i += sprintf(buf, "performance powersave");
 573                 goto out;
 574         }
 575
 576         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 577                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 578                     - (CPUFREQ_NAME_LEN + 2)))
 579                         goto out;
 580                 i += scnprintf(&buf[i], CPUFREQ_NAME_LEN, "%s ", t->name);
 581         }
 582 out:
 583         i += sprintf(&buf[i], "\n");
 584         return i;
 585 }
 586
 587 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 588 {
 589         ssize_t i = 0;
 590         unsigned int cpu;
 591
 592         for_each_cpu(cpu, mask) {
 593                 if (i)
 594                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 595                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 596                 if (i >= (PAGE_SIZE - 5))
 597                         break;
 598         }
 599         i += sprintf(&buf[i], "\n");
 600         return i;
 601 }
 602
 603 /**
 604  * show_related_cpus - show the CPUs affected by each transition even if
 605  * hw coordination is in use
 606  */
 607 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 608 {
 609         if (cpumask_empty(policy->related_cpus))
 610                 return show_cpus(policy->cpus, buf);
 611         return show_cpus(policy->related_cpus, buf);
 612 }
 613
 614 /**
 615  * show_affected_cpus - show the CPUs affected by each transition
 616  */
 617 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 618 {
 619         return show_cpus(policy->cpus, buf);
 620 }
 621
 622 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 623                                         const char *buf, size_t count)
 624 {
 625         unsigned int freq = 0;
 626         unsigned int ret;
 627
 628         if (!policy->governor || !policy->governor->store_setspeed)
 629                 return -EINVAL;
 630
 631         ret = sscanf(buf, "%u", &freq);
 632         if (ret != 1)
 633                 return -EINVAL;
 634
 635         policy->governor->store_setspeed(policy, freq);
 636
 637         return count;
 638 }
 639
 640 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 641 {
 642         if (!policy->governor || !policy->governor->show_setspeed)
 643                 return sprintf(buf, "<unsupported>\n");
 644
 645         return policy->governor->show_setspeed(policy, buf);
 646 }
 647
 648 #define define_one_ro(_name) \
 649 static struct freq_attr _name = \
 650 __ATTR(_name, 0444, show_##_name, NULL)
 651
 652 #define define_one_ro0400(_name) \
 653 static struct freq_attr _name = \
 654 __ATTR(_name, 0400, show_##_name, NULL)
 655
 656 #define define_one_rw(_name) \
 657 static struct freq_attr _name = \
 658 __ATTR(_name, 0644, show_##_name, store_##_name)
 659
 660 define_one_ro0400(cpuinfo_cur_freq);
 661 define_one_ro(cpuinfo_min_freq);
 662 define_one_ro(cpuinfo_max_freq);
 663 define_one_ro(cpuinfo_transition_latency);
 664 define_one_ro(scaling_available_governors);
 665 define_one_ro(scaling_driver);
 666 define_one_ro(scaling_cur_freq);
 667 define_one_ro(related_cpus);
 668 define_one_ro(affected_cpus);
 669 define_one_rw(scaling_min_freq);
 670 define_one_rw(scaling_max_freq);
 671 define_one_rw(scaling_governor);
 672 define_one_rw(scaling_setspeed);
 673
 674 static struct attribute *default_attrs[] = {
 675         &cpuinfo_min_freq.attr,
 676         &cpuinfo_max_freq.attr,
 677         &cpuinfo_transition_latency.attr,
 678         &scaling_min_freq.attr,
 679         &scaling_max_freq.attr,
 680         &affected_cpus.attr,
 681         &related_cpus.attr,
 682         &scaling_governor.attr,
 683         &scaling_driver.attr,
 684         &scaling_available_governors.attr,
 685         &scaling_setspeed.attr,
 686         NULL
 687 };
 688
 689 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 690 #define to_attr(a) container_of(a, struct freq_attr, attr)
 691
 692 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 693 {
 694         struct cpufreq_policy *policy = to_policy(kobj);
 695         struct freq_attr *fattr = to_attr(attr);
 696         ssize_t ret = -EINVAL;
 697         policy = cpufreq_cpu_get(policy->cpu);
 698         if (!policy)
 699                 goto no_policy;
 700
 701         if (lock_policy_rwsem_read(policy->cpu) < 0)
 702                 goto fail;
 703
 704         if (fattr->show)
 705                 ret = fattr->show(policy, buf);
 706         else
 707                 ret = -EIO;
 708
 709         unlock_policy_rwsem_read(policy->cpu);
 710 fail:
 711         cpufreq_cpu_put(policy);
 712 no_policy:
 713         return ret;
 714 }
 715
 716 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 717                      const char *buf, size_t count)
 718 {
 719         struct cpufreq_policy *policy = to_policy(kobj);
 720         struct freq_attr *fattr = to_attr(attr);
 721         ssize_t ret = -EINVAL;
 722         policy = cpufreq_cpu_get(policy->cpu);
 723         if (!policy)
 724                 goto no_policy;
 725
 726         if (lock_policy_rwsem_write(policy->cpu) < 0)
 727                 goto fail;
 728
 729         if (fattr->store)
 730                 ret = fattr->store(policy, buf, count);
 731         else
 732                 ret = -EIO;
 733
 734         unlock_policy_rwsem_write(policy->cpu);
 735 fail:
 736         cpufreq_cpu_put(policy);
 737 no_policy:
 738         return ret;
 739 }
 740
 741 static void cpufreq_sysfs_release(struct kobject *kobj)
 742 {
 743         struct cpufreq_policy *policy = to_policy(kobj);
 744         dprintk("last reference is dropped\n");
 745         complete(&policy->kobj_unregister);
 746 }
 747
 748 static struct sysfs_ops sysfs_ops = {
 749         .show   = show,
 750         .store  = store,
 751 };
 752
 753 static struct kobj_type ktype_cpufreq = {
 754         .sysfs_ops      = &sysfs_ops,
 755         .default_attrs  = default_attrs,
 756         .release        = cpufreq_sysfs_release,
 757 };
 758
 759
 760 int cpufreq_add_dev_policy(unsigned int cpu, struct cpufreq_policy *policy,
 761                 struct sys_device *sys_dev)
 762 {
 763         int ret = 0;
 764 #ifdef CONFIG_SMP
 765         unsigned long flags;
 766         unsigned int j;
 767
 768 #ifdef CONFIG_HOTPLUG_CPU
 769         if (per_cpu(cpufreq_cpu_governor, cpu)) {
 770                 policy->governor = per_cpu(cpufreq_cpu_governor, cpu);
 771                 dprintk("Restoring governor %s for cpu %d\n",
 772                        policy->governor->name, cpu);
 773         }
 774 #endif
 775
 776         for_each_cpu(j, policy->cpus) {
 777                 struct cpufreq_policy *managed_policy;
 778
 779                 if (cpu == j)
 780                         continue;
 781
 782                 /* Check for existing affected CPUs.
 783                  * They may not be aware of it due to CPU Hotplug.
 784                  * cpufreq_cpu_put is called when the device is removed
 785                  * in __cpufreq_remove_dev()
 786                  */
 787                 managed_policy = cpufreq_cpu_get(j);
 788                 if (unlikely(managed_policy)) {
 789
 790                         /* Set proper policy_cpu */
 791                         unlock_policy_rwsem_write(cpu);
 792                         per_cpu(policy_cpu, cpu) = managed_policy->cpu;
 793
 794                         if (lock_policy_rwsem_write(cpu) < 0) {
 795                                 /* Should not go through policy unlock path */
 796                                 if (cpufreq_driver->exit)
 797                                         cpufreq_driver->exit(policy);
 798                                 cpufreq_cpu_put(managed_policy);
 799                                 return -EBUSY;
 800                         }
 801
 802                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 803                         cpumask_copy(managed_policy->cpus, policy->cpus);
 804                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 805                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 806
 807                         dprintk("CPU already managed, adding link\n");
 808                         ret = sysfs_create_link(&sys_dev->kobj,
 809                                                 &managed_policy->kobj,
 810                                                 "cpufreq");
 811                         if (ret)
 812                                 cpufreq_cpu_put(managed_policy);
 813                         /*
 814                          * Success. We only needed to be added to the mask.
 815                          * Call driver->exit() because only the cpu parent of
 816                          * the kobj needed to call init().
 817                          */
 818                         if (cpufreq_driver->exit)
 819                                 cpufreq_driver->exit(policy);
 820                         return ret;
 821                 }
 822         }
 823 #endif
 824         return ret;
 825 }
 826
 827
 828 /* symlink affected CPUs */
 829 int cpufreq_add_dev_symlink(unsigned int cpu, struct cpufreq_policy *policy)
 830 {
 831         unsigned int j;
 832         int ret = 0;
 833
 834         for_each_cpu(j, policy->cpus) {
 835                 struct cpufreq_policy *managed_policy;
 836                 struct sys_device *cpu_sys_dev;
 837
 838                 if (j == cpu)
 839                         continue;
 840                 if (!cpu_online(j))
 841                         continue;
 842
 843                 dprintk("CPU %u already managed, adding link\n", j);
 844                 managed_policy = cpufreq_cpu_get(cpu);
 845                 cpu_sys_dev = get_cpu_sysdev(j);
 846                 ret = sysfs_create_link(&cpu_sys_dev->kobj, &policy->kobj,
 847                                         "cpufreq");
 848                 if (ret) {
 849                         cpufreq_cpu_put(managed_policy);
 850                         return ret;
 851                 }
 852         }
 853         return ret;
 854 }
 855
 856 int cpufreq_add_dev_interface(unsigned int cpu, struct cpufreq_policy *policy,
 857                 struct sys_device *sys_dev)
 858 {
 859         struct cpufreq_policy new_policy;
 860         struct freq_attr **drv_attr;
 861         unsigned long flags;
 862         int ret = 0;
 863         unsigned int j;
 864
 865         /* prepare interface data */
 866         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 867                                    &sys_dev->kobj, "cpufreq");
 868         if (ret)
 869                 return ret;
 870
 871         /* set up files for this cpu device */
 872         drv_attr = cpufreq_driver->attr;
 873         while ((drv_attr) && (*drv_attr)) {
 874                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 875                 if (ret)
 876                         goto err_out_kobj_put;
 877                 drv_attr++;
 878         }
 879         if (cpufreq_driver->get) {
 880                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 881                 if (ret)
 882                         goto err_out_kobj_put;
 883         }
 884         if (cpufreq_driver->target) {
 885                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 886                 if (ret)
 887                         goto err_out_kobj_put;
 888         }
 889
 890         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 891         for_each_cpu(j, policy->cpus) {
 892         if (!cpu_online(j))
 893                 continue;
 894                 per_cpu(cpufreq_cpu_data, j) = policy;
 895                 per_cpu(policy_cpu, j) = policy->cpu;
 896         }
 897         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 898
 899         ret = cpufreq_add_dev_symlink(cpu, policy);
 900         if (ret)
 901                 goto err_out_kobj_put;
 902
 903         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 904         /* assure that the starting sequence is run in __cpufreq_set_policy */
 905         policy->governor = NULL;
 906
 907         /* set default policy */
 908         ret = __cpufreq_set_policy(policy, &new_policy);
 909         policy->user_policy.policy = policy->policy;
 910         policy->user_policy.governor = policy->governor;
 911
 912         if (ret) {
 913                 dprintk("setting policy failed\n");
 914                 if (cpufreq_driver->exit)
 915                         cpufreq_driver->exit(policy);
 916         }
 917         return ret;
 918
 919 err_out_kobj_put:
 920         kobject_put(&policy->kobj);
 921         wait_for_completion(&policy->kobj_unregister);
 922         return ret;
 923 }
 924
 925
 926 /**
 927  * cpufreq_add_dev - add a CPU device
 928  *
 929  * Adds the cpufreq interface for a CPU device.
 930  *
 931  * The Oracle says: try running cpufreq registration/unregistration concurrently
 932  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 933  * mess up, but more thorough testing is needed. - Mathieu
 934  */
 935 static int cpufreq_add_dev(struct sys_device *sys_dev)
 936 {
 937         unsigned int cpu = sys_dev->id;
 938         int ret = 0;
 939         struct cpufreq_policy *policy;
 940         unsigned long flags;
 941         unsigned int j;
 942
 943         if (cpu_is_offline(cpu))
 944                 return 0;
 945
 946         cpufreq_debug_disable_ratelimit();
 947         dprintk("adding CPU %u\n", cpu);
 948
 949 #ifdef CONFIG_SMP
 950         /* check whether a different CPU already registered this
 951          * CPU because it is in the same boat. */
 952         policy = cpufreq_cpu_get(cpu);
 953         if (unlikely(policy)) {
 954                 cpufreq_cpu_put(policy);
 955                 cpufreq_debug_enable_ratelimit();
 956                 return 0;
 957         }
 958 #endif
 959
 960         if (!try_module_get(cpufreq_driver->owner)) {
 961                 ret = -EINVAL;
 962                 goto module_out;
 963         }
 964
 965         ret = -ENOMEM;
 966         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 967         if (!policy)
 968                 goto nomem_out;
 969
 970         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 971                 goto err_free_policy;
 972
 973         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 974                 goto err_free_cpumask;
 975
 976         policy->cpu = cpu;
 977         cpumask_copy(policy->cpus, cpumask_of(cpu));
 978
 979         /* Initially set CPU itself as the policy_cpu */
 980         per_cpu(policy_cpu, cpu) = cpu;
 981         ret = (lock_policy_rwsem_write(cpu) < 0);
 982         WARN_ON(ret);
 983
 984         init_completion(&policy->kobj_unregister);
 985         INIT_WORK(&policy->update, handle_update);
 986
 987         /* Set governor before ->init, so that driver could check it */
 988         policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 989         /* call driver. From then on the cpufreq must be able
 990          * to accept all calls to ->verify and ->setpolicy for this CPU
 991          */
 992         ret = cpufreq_driver->init(policy);
 993         if (ret) {
 994                 dprintk("initialization failed\n");
 995                 goto err_unlock_policy;
 996         }
 997         policy->user_policy.min = policy->min;
 998         policy->user_policy.max = policy->max;
 999
1000         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1001                                      CPUFREQ_START, policy);
1002
1003         ret = cpufreq_add_dev_policy(cpu, policy, sys_dev);
1004         if (ret)
1005                 goto err_unlock_policy;
1006
1007         ret = cpufreq_add_dev_interface(cpu, policy, sys_dev);
1008         if (ret)
1009                 goto err_out_unregister;
1010
1011         unlock_policy_rwsem_write(cpu);
1012
1013         kobject_uevent(&policy->kobj, KOBJ_ADD);
1014         module_put(cpufreq_driver->owner);
1015         dprintk("initialization complete\n");
1016         cpufreq_debug_enable_ratelimit();
1017
1018         return 0;
1019
1020
1021 err_out_unregister:
1022         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1023         for_each_cpu(j, policy->cpus)
1024                 per_cpu(cpufreq_cpu_data, j) = NULL;
1025         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1026
1027         kobject_put(&policy->kobj);
1028         wait_for_completion(&policy->kobj_unregister);
1029
1030 err_unlock_policy:
1031         unlock_policy_rwsem_write(cpu);
1032 err_free_cpumask:
1033         free_cpumask_var(policy->cpus);
1034 err_free_policy:
1035         kfree(policy);
1036 nomem_out:
1037         module_put(cpufreq_driver->owner);
1038 module_out:
1039         cpufreq_debug_enable_ratelimit();
1040         return ret;
1041 }
1042
1043
1044 /**
1045  * __cpufreq_remove_dev - remove a CPU device
1046  *
1047  * Removes the cpufreq interface for a CPU device.
1048  * Caller should already have policy_rwsem in write mode for this CPU.
1049  * This routine frees the rwsem before returning.
1050  */
1051 static int __cpufreq_remove_dev(struct sys_device *sys_dev)
1052 {
1053         unsigned int cpu = sys_dev->id;
1054         unsigned long flags;
1055         struct cpufreq_policy *data;
1056 #ifdef CONFIG_SMP
1057         struct sys_device *cpu_sys_dev;
1058         unsigned int j;
1059 #endif
1060
1061         cpufreq_debug_disable_ratelimit();
1062         dprintk("unregistering CPU %u\n", cpu);
1063
1064         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1065         data = per_cpu(cpufreq_cpu_data, cpu);
1066
1067         if (!data) {
1068                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1069                 cpufreq_debug_enable_ratelimit();
1070                 unlock_policy_rwsem_write(cpu);
1071                 return -EINVAL;
1072         }
1073         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1074
1075
1076 #ifdef CONFIG_SMP
1077         /* if this isn't the CPU which is the parent of the kobj, we
1078          * only need to unlink, put and exit
1079          */
1080         if (unlikely(cpu != data->cpu)) {
1081                 dprintk("removing link\n");
1082                 cpumask_clear_cpu(cpu, data->cpus);
1083                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1084                 sysfs_remove_link(&sys_dev->kobj, "cpufreq");
1085                 cpufreq_cpu_put(data);
1086                 cpufreq_debug_enable_ratelimit();
1087                 unlock_policy_rwsem_write(cpu);
1088                 return 0;
1089         }
1090 #endif
1091
1092 #ifdef CONFIG_SMP
1093
1094 #ifdef CONFIG_HOTPLUG_CPU
1095         per_cpu(cpufreq_cpu_governor, cpu) = data->governor;
1096 #endif
1097
1098         /* if we have other CPUs still registered, we need to unlink them,
1099          * or else wait_for_completion below will lock up. Clean the
1100          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1101          * the sysfs links afterwards.
1102          */
1103         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1104                 for_each_cpu(j, data->cpus) {
1105                         if (j == cpu)
1106                                 continue;
1107                         per_cpu(cpufreq_cpu_data, j) = NULL;
1108                 }
1109         }
1110
1111         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1112
1113         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1114                 for_each_cpu(j, data->cpus) {
1115                         if (j == cpu)
1116                                 continue;
1117                         dprintk("removing link for cpu %u\n", j);
1118 #ifdef CONFIG_HOTPLUG_CPU
1119                         per_cpu(cpufreq_cpu_governor, j) = data->governor;
1120 #endif
1121                         cpu_sys_dev = get_cpu_sysdev(j);
1122                         sysfs_remove_link(&cpu_sys_dev->kobj, "cpufreq");
1123                         cpufreq_cpu_put(data);
1124                 }
1125         }
1126 #else
1127         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1128 #endif
1129
1130         if (cpufreq_driver->target)
1131                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1132
1133         kobject_put(&data->kobj);
1134
1135         /* we need to make sure that the underlying kobj is actually
1136          * not referenced anymore by anybody before we proceed with
1137          * unloading.
1138          */
1139         dprintk("waiting for dropping of refcount\n");
1140         wait_for_completion(&data->kobj_unregister);
1141         dprintk("wait complete\n");
1142
1143         if (cpufreq_driver->exit)
1144                 cpufreq_driver->exit(data);
1145
1146         unlock_policy_rwsem_write(cpu);
1147
1148         free_cpumask_var(data->related_cpus);
1149         free_cpumask_var(data->cpus);
1150         kfree(data);
1151         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1152
1153         cpufreq_debug_enable_ratelimit();
1154         return 0;
1155 }
1156
1157
1158 static int cpufreq_remove_dev(struct sys_device *sys_dev)
1159 {
1160         unsigned int cpu = sys_dev->id;
1161         int retval;
1162
1163         if (cpu_is_offline(cpu))
1164                 return 0;
1165
1166         if (unlikely(lock_policy_rwsem_write(cpu)))
1167                 BUG();
1168
1169         retval = __cpufreq_remove_dev(sys_dev);
1170         return retval;
1171 }
1172
1173
1174 static void handle_update(struct work_struct *work)
1175 {
1176         struct cpufreq_policy *policy =
1177                 container_of(work, struct cpufreq_policy, update);
1178         unsigned int cpu = policy->cpu;
1179         dprintk("handle_update for cpu %u called\n", cpu);
1180         cpufreq_update_policy(cpu);
1181 }
1182
1183 /**
1184  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1185  *      @cpu: cpu number
1186  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1187  *      @new_freq: CPU frequency the CPU actually runs at
1188  *
1189  *      We adjust to current frequency first, and need to clean up later.
1190  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1191  */
1192 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1193                                 unsigned int new_freq)
1194 {
1195         struct cpufreq_freqs freqs;
1196
1197         dprintk("Warning: CPU frequency out of sync: cpufreq and timing "
1198                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1199
1200         freqs.cpu = cpu;
1201         freqs.old = old_freq;
1202         freqs.new = new_freq;
1203         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1204         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1205 }
1206
1207
1208 /**
1209  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1210  * @cpu: CPU number
1211  *
1212  * This is the last known freq, without actually getting it from the driver.
1213  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1214  */
1215 unsigned int cpufreq_quick_get(unsigned int cpu)
1216 {
1217         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1218         unsigned int ret_freq = 0;
1219
1220         if (policy) {
1221                 ret_freq = policy->cur;
1222                 cpufreq_cpu_put(policy);
1223         }
1224
1225         return ret_freq;
1226 }
1227 EXPORT_SYMBOL(cpufreq_quick_get);
1228
1229
1230 static unsigned int __cpufreq_get(unsigned int cpu)
1231 {
1232         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1233         unsigned int ret_freq = 0;
1234
1235         if (!cpufreq_driver->get)
1236                 return ret_freq;
1237
1238         ret_freq = cpufreq_driver->get(cpu);
1239
1240         if (ret_freq && policy->cur &&
1241                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1242                 /* verify no discrepancy between actual and
1243                                         saved value exists */
1244                 if (unlikely(ret_freq != policy->cur)) {
1245                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1246                         schedule_work(&policy->update);
1247                 }
1248         }
1249
1250         return ret_freq;
1251 }
1252
1253 /**
1254  * cpufreq_get - get the current CPU frequency (in kHz)
1255  * @cpu: CPU number
1256  *
1257  * Get the CPU current (static) CPU frequency
1258  */
1259 unsigned int cpufreq_get(unsigned int cpu)
1260 {
1261         unsigned int ret_freq = 0;
1262         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1263
1264         if (!policy)
1265                 goto out;
1266
1267         if (unlikely(lock_policy_rwsem_read(cpu)))
1268                 goto out_policy;
1269
1270         ret_freq = __cpufreq_get(cpu);
1271
1272         unlock_policy_rwsem_read(cpu);
1273
1274 out_policy:
1275         cpufreq_cpu_put(policy);
1276 out:
1277         return ret_freq;
1278 }
1279 EXPORT_SYMBOL(cpufreq_get);
1280
1281
1282 /**
1283  *      cpufreq_suspend - let the low level driver prepare for suspend
1284  */
1285
1286 static int cpufreq_suspend(struct sys_device *sysdev, pm_message_t pmsg)
1287 {
1288         int ret = 0;
1289
1290         int cpu = sysdev->id;
1291         struct cpufreq_policy *cpu_policy;
1292
1293         dprintk("suspending cpu %u\n", cpu);
1294
1295         if (!cpu_online(cpu))
1296                 return 0;
1297
1298         /* we may be lax here as interrupts are off. Nonetheless
1299          * we need to grab the correct cpu policy, as to check
1300          * whether we really run on this CPU.
1301          */
1302
1303         cpu_policy = cpufreq_cpu_get(cpu);
1304         if (!cpu_policy)
1305                 return -EINVAL;
1306
1307         /* only handle each CPU group once */
1308         if (unlikely(cpu_policy->cpu != cpu))
1309                 goto out;
1310
1311         if (cpufreq_driver->suspend) {
1312                 ret = cpufreq_driver->suspend(cpu_policy, pmsg);
1313                 if (ret)
1314                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1315                                         "step on CPU %u\n", cpu_policy->cpu);
1316         }
1317
1318 out:
1319         cpufreq_cpu_put(cpu_policy);
1320         return ret;
1321 }
1322
1323 /**
1324  *      cpufreq_resume -  restore proper CPU frequency handling after resume
1325  *
1326  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1327  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1328  *          restored. It will verify that the current freq is in sync with
1329  *          what we believe it to be. This is a bit later than when it
1330  *          should be, but nonethteless it's better than calling
1331  *          cpufreq_driver->get() here which might re-enable interrupts...
1332  */
1333 static int cpufreq_resume(struct sys_device *sysdev)
1334 {
1335         int ret = 0;
1336
1337         int cpu = sysdev->id;
1338         struct cpufreq_policy *cpu_policy;
1339
1340         dprintk("resuming cpu %u\n", cpu);
1341
1342         if (!cpu_online(cpu))
1343                 return 0;
1344
1345         /* we may be lax here as interrupts are off. Nonetheless
1346          * we need to grab the correct cpu policy, as to check
1347          * whether we really run on this CPU.
1348          */
1349
1350         cpu_policy = cpufreq_cpu_get(cpu);
1351         if (!cpu_policy)
1352                 return -EINVAL;
1353
1354         /* only handle each CPU group once */
1355         if (unlikely(cpu_policy->cpu != cpu))
1356                 goto fail;
1357
1358         if (cpufreq_driver->resume) {
1359                 ret = cpufreq_driver->resume(cpu_policy);
1360                 if (ret) {
1361                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1362                                         "step on CPU %u\n", cpu_policy->cpu);
1363                         goto fail;
1364                 }
1365         }
1366
1367         schedule_work(&cpu_policy->update);
1368
1369 fail:
1370         cpufreq_cpu_put(cpu_policy);
1371         return ret;
1372 }
1373
1374 static struct sysdev_driver cpufreq_sysdev_driver = {
1375         .add            = cpufreq_add_dev,
1376         .remove         = cpufreq_remove_dev,
1377         .suspend        = cpufreq_suspend,
1378         .resume         = cpufreq_resume,
1379 };
1380
1381
1382 /*********************************************************************
1383  *                     NOTIFIER LISTS INTERFACE                      *
1384  *********************************************************************/
1385
1386 /**
1387  *      cpufreq_register_notifier - register a driver with cpufreq
1388  *      @nb: notifier function to register
1389  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1390  *
1391  *      Add a driver to one of two lists: either a list of drivers that
1392  *      are notified about clock rate changes (once before and once after
1393  *      the transition), or a list of drivers that are notified about
1394  *      changes in cpufreq policy.
1395  *
1396  *      This function may sleep, and has the same return conditions as
1397  *      blocking_notifier_chain_register.
1398  */
1399 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1400 {
1401         int ret;
1402
1403         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1404
1405         switch (list) {
1406         case CPUFREQ_TRANSITION_NOTIFIER:
1407                 ret = srcu_notifier_chain_register(
1408                                 &cpufreq_transition_notifier_list, nb);
1409                 break;
1410         case CPUFREQ_POLICY_NOTIFIER:
1411                 ret = blocking_notifier_chain_register(
1412                                 &cpufreq_policy_notifier_list, nb);
1413                 break;
1414         default:
1415                 ret = -EINVAL;
1416         }
1417
1418         return ret;
1419 }
1420 EXPORT_SYMBOL(cpufreq_register_notifier);
1421
1422
1423 /**
1424  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1425  *      @nb: notifier block to be unregistered
1426  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1427  *
1428  *      Remove a driver from the CPU frequency notifier list.
1429  *
1430  *      This function may sleep, and has the same return conditions as
1431  *      blocking_notifier_chain_unregister.
1432  */
1433 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1434 {
1435         int ret;
1436
1437         switch (list) {
1438         case CPUFREQ_TRANSITION_NOTIFIER:
1439                 ret = srcu_notifier_chain_unregister(
1440                                 &cpufreq_transition_notifier_list, nb);
1441                 break;
1442         case CPUFREQ_POLICY_NOTIFIER:
1443                 ret = blocking_notifier_chain_unregister(
1444                                 &cpufreq_policy_notifier_list, nb);
1445                 break;
1446         default:
1447                 ret = -EINVAL;
1448         }
1449
1450         return ret;
1451 }
1452 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1453
1454
1455 /*********************************************************************
1456  *                              GOVERNORS                            *
1457  *********************************************************************/
1458
1459
1460 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1461                             unsigned int target_freq,
1462                             unsigned int relation)
1463 {
1464         int retval = -EINVAL;
1465
1466         dprintk("target for CPU %u: %u kHz, relation %u\n", policy->cpu,
1467                 target_freq, relation);
1468         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1469                 retval = cpufreq_driver->target(policy, target_freq, relation);
1470
1471         return retval;
1472 }
1473 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1474
1475 int cpufreq_driver_target(struct cpufreq_policy *policy,
1476                           unsigned int target_freq,
1477                           unsigned int relation)
1478 {
1479         int ret = -EINVAL;
1480
1481         policy = cpufreq_cpu_get(policy->cpu);
1482         if (!policy)
1483                 goto no_policy;
1484
1485         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1486                 goto fail;
1487
1488         ret = __cpufreq_driver_target(policy, target_freq, relation);
1489
1490         unlock_policy_rwsem_write(policy->cpu);
1491
1492 fail:
1493         cpufreq_cpu_put(policy);
1494 no_policy:
1495         return ret;
1496 }
1497 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1498
1499 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1500 {
1501         int ret = 0;
1502
1503         policy = cpufreq_cpu_get(policy->cpu);
1504         if (!policy)
1505                 return -EINVAL;
1506
1507         if (cpu_online(cpu) && cpufreq_driver->getavg)
1508                 ret = cpufreq_driver->getavg(policy, cpu);
1509
1510         cpufreq_cpu_put(policy);
1511         return ret;
1512 }
1513 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1514
1515 /*
1516  * when "event" is CPUFREQ_GOV_LIMITS
1517  */
1518
1519 static int __cpufreq_governor(struct cpufreq_policy *policy,
1520                                         unsigned int event)
1521 {
1522         int ret;
1523
1524         /* Only must be defined when default governor is known to have latency
1525            restrictions, like e.g. conservative or ondemand.
1526            That this is the case is already ensured in Kconfig
1527         */
1528 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1529         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1530 #else
1531         struct cpufreq_governor *gov = NULL;
1532 #endif
1533
1534         if (policy->governor->max_transition_latency &&
1535             policy->cpuinfo.transition_latency >
1536             policy->governor->max_transition_latency) {
1537                 if (!gov)
1538                         return -EINVAL;
1539                 else {
1540                         printk(KERN_WARNING "%s governor failed, too long"
1541                                " transition latency of HW, fallback"
1542                                " to %s governor\n",
1543                                policy->governor->name,
1544                                gov->name);
1545                         policy->governor = gov;
1546                 }
1547         }
1548
1549         if (!try_module_get(policy->governor->owner))
1550                 return -EINVAL;
1551
1552         dprintk("__cpufreq_governor for CPU %u, event %u\n",
1553                                                 policy->cpu, event);
1554         ret = policy->governor->governor(policy, event);
1555
1556         /* we keep one module reference alive for
1557                         each CPU governed by this CPU */
1558         if ((event != CPUFREQ_GOV_START) || ret)
1559                 module_put(policy->governor->owner);
1560         if ((event == CPUFREQ_GOV_STOP) && !ret)
1561                 module_put(policy->governor->owner);
1562
1563         return ret;
1564 }
1565
1566
1567 int cpufreq_register_governor(struct cpufreq_governor *governor)
1568 {
1569         int err;
1570
1571         if (!governor)
1572                 return -EINVAL;
1573
1574         mutex_lock(&cpufreq_governor_mutex);
1575
1576         err = -EBUSY;
1577         if (__find_governor(governor->name) == NULL) {
1578                 err = 0;
1579                 list_add(&governor->governor_list, &cpufreq_governor_list);
1580         }
1581
1582         mutex_unlock(&cpufreq_governor_mutex);
1583         return err;
1584 }
1585 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1586
1587
1588 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1589 {
1590         if (!governor)
1591                 return;
1592
1593         mutex_lock(&cpufreq_governor_mutex);
1594         list_del(&governor->governor_list);
1595         mutex_unlock(&cpufreq_governor_mutex);
1596         return;
1597 }
1598 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1599
1600
1601
1602 /*********************************************************************
1603  *                          POLICY INTERFACE                         *
1604  *********************************************************************/
1605
1606 /**
1607  * cpufreq_get_policy - get the current cpufreq_policy
1608  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1609  *      is written
1610  *
1611  * Reads the current cpufreq policy.
1612  */
1613 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1614 {
1615         struct cpufreq_policy *cpu_policy;
1616         if (!policy)
1617                 return -EINVAL;
1618
1619         cpu_policy = cpufreq_cpu_get(cpu);
1620         if (!cpu_policy)
1621                 return -EINVAL;
1622
1623         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1624
1625         cpufreq_cpu_put(cpu_policy);
1626         return 0;
1627 }
1628 EXPORT_SYMBOL(cpufreq_get_policy);
1629
1630
1631 /*
1632  * data   : current policy.
1633  * policy : policy to be set.
1634  */
1635 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1636                                 struct cpufreq_policy *policy)
1637 {
1638         int ret = 0;
1639
1640         cpufreq_debug_disable_ratelimit();
1641         dprintk("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1642                 policy->min, policy->max);
1643
1644         memcpy(&policy->cpuinfo, &data->cpuinfo,
1645                                 sizeof(struct cpufreq_cpuinfo));
1646
1647         if (policy->min > data->max || policy->max < data->min) {
1648                 ret = -EINVAL;
1649                 goto error_out;
1650         }
1651
1652         /* verify the cpu speed can be set within this limit */
1653         ret = cpufreq_driver->verify(policy);
1654         if (ret)
1655                 goto error_out;
1656
1657         /* adjust if necessary - all reasons */
1658         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1659                         CPUFREQ_ADJUST, policy);
1660
1661         /* adjust if necessary - hardware incompatibility*/
1662         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1663                         CPUFREQ_INCOMPATIBLE, policy);
1664
1665         /* verify the cpu speed can be set within this limit,
1666            which might be different to the first one */
1667         ret = cpufreq_driver->verify(policy);
1668         if (ret)
1669                 goto error_out;
1670
1671         /* notification of the new policy */
1672         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1673                         CPUFREQ_NOTIFY, policy);
1674
1675         data->min = policy->min;
1676         data->max = policy->max;
1677
1678         dprintk("new min and max freqs are %u - %u kHz\n",
1679                                         data->min, data->max);
1680
1681         if (cpufreq_driver->setpolicy) {
1682                 data->policy = policy->policy;
1683                 dprintk("setting range\n");
1684                 ret = cpufreq_driver->setpolicy(policy);
1685         } else {
1686                 if (policy->governor != data->governor) {
1687                         /* save old, working values */
1688                         struct cpufreq_governor *old_gov = data->governor;
1689
1690                         dprintk("governor switch\n");
1691
1692                         /* end old governor */
1693                         if (data->governor)
1694                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1695
1696                         /* start new governor */
1697                         data->governor = policy->governor;
1698                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1699                                 /* new governor failed, so re-start old one */
1700                                 dprintk("starting governor %s failed\n",
1701                                                         data->governor->name);
1702                                 if (old_gov) {
1703                                         data->governor = old_gov;
1704                                         __cpufreq_governor(data,
1705                                                            CPUFREQ_GOV_START);
1706                                 }
1707                                 ret = -EINVAL;
1708                                 goto error_out;
1709                         }
1710                         /* might be a policy change, too, so fall through */
1711                 }
1712                 dprintk("governor: change or update limits\n");
1713                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1714         }
1715
1716 error_out:
1717         cpufreq_debug_enable_ratelimit();
1718         return ret;
1719 }
1720
1721 /**
1722  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1723  *      @cpu: CPU which shall be re-evaluated
1724  *
1725  *      Usefull for policy notifiers which have different necessities
1726  *      at different times.
1727  */
1728 int cpufreq_update_policy(unsigned int cpu)
1729 {
1730         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1731         struct cpufreq_policy policy;
1732         int ret;
1733
1734         if (!data) {
1735                 ret = -ENODEV;
1736                 goto no_policy;
1737         }
1738
1739         if (unlikely(lock_policy_rwsem_write(cpu))) {
1740                 ret = -EINVAL;
1741                 goto fail;
1742         }
1743
1744         dprintk("updating policy for CPU %u\n", cpu);
1745         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1746         policy.min = data->user_policy.min;
1747         policy.max = data->user_policy.max;
1748         policy.policy = data->user_policy.policy;
1749         policy.governor = data->user_policy.governor;
1750
1751         /* BIOS might change freq behind our back
1752           -> ask driver for current freq and notify governors about a change */
1753         if (cpufreq_driver->get) {
1754                 policy.cur = cpufreq_driver->get(cpu);
1755                 if (!data->cur) {
1756                         dprintk("Driver did not initialize current freq");
1757                         data->cur = policy.cur;
1758                 } else {
1759                         if (data->cur != policy.cur)
1760                                 cpufreq_out_of_sync(cpu, data->cur,
1761                                                                 policy.cur);
1762                 }
1763         }
1764
1765         ret = __cpufreq_set_policy(data, &policy);
1766
1767         unlock_policy_rwsem_write(cpu);
1768
1769 fail:
1770         cpufreq_cpu_put(data);
1771 no_policy:
1772         return ret;
1773 }
1774 EXPORT_SYMBOL(cpufreq_update_policy);
1775
1776 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1777                                         unsigned long action, void *hcpu)
1778 {
1779         unsigned int cpu = (unsigned long)hcpu;
1780         struct sys_device *sys_dev;
1781
1782         sys_dev = get_cpu_sysdev(cpu);
1783         if (sys_dev) {
1784                 switch (action) {
1785                 case CPU_ONLINE:
1786                 case CPU_ONLINE_FROZEN:
1787                         cpufreq_add_dev(sys_dev);
1788                         break;
1789                 case CPU_DOWN_PREPARE:
1790                 case CPU_DOWN_PREPARE_FROZEN:
1791                         if (unlikely(lock_policy_rwsem_write(cpu)))
1792                                 BUG();
1793
1794                         __cpufreq_remove_dev(sys_dev);
1795                         break;
1796                 case CPU_DOWN_FAILED:
1797                 case CPU_DOWN_FAILED_FROZEN:
1798                         cpufreq_add_dev(sys_dev);
1799                         break;
1800                 }
1801         }
1802         return NOTIFY_OK;
1803 }
1804
1805 static struct notifier_block __refdata cpufreq_cpu_notifier =
1806 {
1807     .notifier_call = cpufreq_cpu_callback,
1808 };
1809
1810 /*********************************************************************
1811  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1812  *********************************************************************/
1813
1814 /**
1815  * cpufreq_register_driver - register a CPU Frequency driver
1816  * @driver_data: A struct cpufreq_driver containing the values#
1817  * submitted by the CPU Frequency driver.
1818  *
1819  *   Registers a CPU Frequency driver to this core code. This code
1820  * returns zero on success, -EBUSY when another driver got here first
1821  * (and isn't unregistered in the meantime).
1822  *
1823  */
1824 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1825 {
1826         unsigned long flags;
1827         int ret;
1828
1829         if (!driver_data || !driver_data->verify || !driver_data->init ||
1830             ((!driver_data->setpolicy) && (!driver_data->target)))
1831                 return -EINVAL;
1832
1833         dprintk("trying to register driver %s\n", driver_data->name);
1834
1835         if (driver_data->setpolicy)
1836                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1837
1838         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1839         if (cpufreq_driver) {
1840                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1841                 return -EBUSY;
1842         }
1843         cpufreq_driver = driver_data;
1844         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1845
1846         ret = sysdev_driver_register(&cpu_sysdev_class,
1847                                         &cpufreq_sysdev_driver);
1848
1849         if ((!ret) && !(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1850                 int i;
1851                 ret = -ENODEV;
1852
1853                 /* check for at least one working CPU */
1854                 for (i = 0; i < nr_cpu_ids; i++)
1855                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1856                                 ret = 0;
1857                                 break;
1858                         }
1859
1860                 /* if all ->init() calls failed, unregister */
1861                 if (ret) {
1862                         dprintk("no CPU initialized for driver %s\n",
1863                                                         driver_data->name);
1864                         sysdev_driver_unregister(&cpu_sysdev_class,
1865                                                 &cpufreq_sysdev_driver);
1866
1867                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1868                         cpufreq_driver = NULL;
1869                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1870                 }
1871         }
1872
1873         if (!ret) {
1874                 register_hotcpu_notifier(&cpufreq_cpu_notifier);
1875                 dprintk("driver %s up and running\n", driver_data->name);
1876                 cpufreq_debug_enable_ratelimit();
1877         }
1878
1879         return ret;
1880 }
1881 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1882
1883
1884 /**
1885  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1886  *
1887  *    Unregister the current CPUFreq driver. Only call this if you have
1888  * the right to do so, i.e. if you have succeeded in initialising before!
1889  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1890  * currently not initialised.
1891  */
1892 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1893 {
1894         unsigned long flags;
1895
1896         cpufreq_debug_disable_ratelimit();
1897
1898         if (!cpufreq_driver || (driver != cpufreq_driver)) {
1899                 cpufreq_debug_enable_ratelimit();
1900                 return -EINVAL;
1901         }
1902
1903         dprintk("unregistering driver %s\n", driver->name);
1904
1905         sysdev_driver_unregister(&cpu_sysdev_class, &cpufreq_sysdev_driver);
1906         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1907
1908         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1909         cpufreq_driver = NULL;
1910         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1911
1912         return 0;
1913 }
1914 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1915
1916 static int __init cpufreq_core_init(void)
1917 {
1918         int cpu;
1919
1920         for_each_possible_cpu(cpu) {
1921                 per_cpu(policy_cpu, cpu) = -1;
1922                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1923         }
1924         return 0;
1925 }
1926
1927 core_initcall(cpufreq_core_init);