2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIe link L0s/L1 state and Clock Power Management
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
29 /* Note: those are not register definitions */
30 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
31 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
32 #define ASPM_STATE_L1 (4) /* L1 state */
33 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
34 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1)
37 u32 l0s; /* L0s latency (nsec) */
38 u32 l1; /* L1 latency (nsec) */
41 struct pcie_link_state {
42 struct pci_dev *pdev; /* Upstream component of the Link */
43 struct pcie_link_state *root; /* pointer to the root port link */
44 struct pcie_link_state *parent; /* pointer to the parent Link state */
45 struct list_head sibling; /* node in link_list */
46 struct list_head children; /* list of child link states */
47 struct list_head link; /* node in parent's children list */
50 u32 aspm_support:3; /* Supported ASPM state */
51 u32 aspm_enabled:3; /* Enabled ASPM state */
52 u32 aspm_capable:3; /* Capable ASPM state with latency */
53 u32 aspm_default:3; /* Default ASPM state by BIOS */
54 u32 aspm_disable:3; /* Disabled ASPM state */
57 u32 clkpm_capable:1; /* Clock PM capable? */
58 u32 clkpm_enabled:1; /* Current Clock PM state */
59 u32 clkpm_default:1; /* Default Clock PM state by BIOS */
62 struct aspm_latency latency_up; /* Upstream direction exit latency */
63 struct aspm_latency latency_dw; /* Downstream direction exit latency */
65 * Endpoint acceptable latencies. A pcie downstream port only
66 * has one slot under it, so at most there are 8 functions.
68 struct aspm_latency acceptable[8];
71 static int aspm_disabled, aspm_force;
72 static bool aspm_support_enabled = true;
73 static DEFINE_MUTEX(aspm_lock);
74 static LIST_HEAD(link_list);
76 #define POLICY_DEFAULT 0 /* BIOS default setting */
77 #define POLICY_PERFORMANCE 1 /* high performance */
78 #define POLICY_POWERSAVE 2 /* high power saving */
80 #ifdef CONFIG_PCIEASPM_PERFORMANCE
81 static int aspm_policy = POLICY_PERFORMANCE;
82 #elif defined CONFIG_PCIEASPM_POWERSAVE
83 static int aspm_policy = POLICY_POWERSAVE;
85 static int aspm_policy;
88 static const char *policy_str[] = {
89 [POLICY_DEFAULT] = "default",
90 [POLICY_PERFORMANCE] = "performance",
91 [POLICY_POWERSAVE] = "powersave"
94 #define LINK_RETRAIN_TIMEOUT HZ
96 static int policy_to_aspm_state(struct pcie_link_state *link)
98 switch (aspm_policy) {
99 case POLICY_PERFORMANCE:
100 /* Disable ASPM and Clock PM */
102 case POLICY_POWERSAVE:
103 /* Enable ASPM L0s/L1 */
104 return ASPM_STATE_ALL;
106 return link->aspm_default;
111 static int policy_to_clkpm_state(struct pcie_link_state *link)
113 switch (aspm_policy) {
114 case POLICY_PERFORMANCE:
115 /* Disable ASPM and Clock PM */
117 case POLICY_POWERSAVE:
118 /* Disable Clock PM */
121 return link->clkpm_default;
126 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
130 struct pci_dev *child;
131 struct pci_bus *linkbus = link->pdev->subordinate;
133 list_for_each_entry(child, &linkbus->devices, bus_list) {
134 pos = pci_pcie_cap(child);
137 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
139 reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
141 reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
142 pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
144 link->clkpm_enabled = !!enable;
147 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
149 /* Don't enable Clock PM if the link is not Clock PM capable */
150 if (!link->clkpm_capable && enable)
152 /* Need nothing if the specified equals to current state */
153 if (link->clkpm_enabled == enable)
155 pcie_set_clkpm_nocheck(link, enable);
158 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
160 int pos, capable = 1, enabled = 1;
163 struct pci_dev *child;
164 struct pci_bus *linkbus = link->pdev->subordinate;
166 /* All functions should have the same cap and state, take the worst */
167 list_for_each_entry(child, &linkbus->devices, bus_list) {
168 pos = pci_pcie_cap(child);
171 pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, ®32);
172 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
177 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
178 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
181 link->clkpm_enabled = enabled;
182 link->clkpm_default = enabled;
183 link->clkpm_capable = (blacklist) ? 0 : capable;
187 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
188 * could use common clock. If they are, configure them to use the
189 * common clock. That will reduce the ASPM state exit latency.
191 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
193 int ppos, cpos, same_clock = 1;
194 u16 reg16, parent_reg, child_reg[8];
195 unsigned long start_jiffies;
196 struct pci_dev *child, *parent = link->pdev;
197 struct pci_bus *linkbus = parent->subordinate;
199 * All functions of a slot should have the same Slot Clock
200 * Configuration, so just check one function
202 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
203 BUG_ON(!pci_is_pcie(child));
205 /* Check downstream component if bit Slot Clock Configuration is 1 */
206 cpos = pci_pcie_cap(child);
207 pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, ®16);
208 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
211 /* Check upstream component if bit Slot Clock Configuration is 1 */
212 ppos = pci_pcie_cap(parent);
213 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
214 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
217 /* Configure downstream component, all functions */
218 list_for_each_entry(child, &linkbus->devices, bus_list) {
219 cpos = pci_pcie_cap(child);
220 pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, ®16);
221 child_reg[PCI_FUNC(child->devfn)] = reg16;
223 reg16 |= PCI_EXP_LNKCTL_CCC;
225 reg16 &= ~PCI_EXP_LNKCTL_CCC;
226 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
229 /* Configure upstream component */
230 pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, ®16);
233 reg16 |= PCI_EXP_LNKCTL_CCC;
235 reg16 &= ~PCI_EXP_LNKCTL_CCC;
236 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
239 reg16 |= PCI_EXP_LNKCTL_RL;
240 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
242 /* Wait for link training end. Break out after waiting for timeout */
243 start_jiffies = jiffies;
245 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
246 if (!(reg16 & PCI_EXP_LNKSTA_LT))
248 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
252 if (!(reg16 & PCI_EXP_LNKSTA_LT))
255 /* Training failed. Restore common clock configurations */
256 dev_printk(KERN_ERR, &parent->dev,
257 "ASPM: Could not configure common clock\n");
258 list_for_each_entry(child, &linkbus->devices, bus_list) {
259 cpos = pci_pcie_cap(child);
260 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
261 child_reg[PCI_FUNC(child->devfn)]);
263 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
266 /* Convert L0s latency encoding to ns */
267 static u32 calc_l0s_latency(u32 encoding)
270 return (5 * 1000); /* > 4us */
271 return (64 << encoding);
274 /* Convert L0s acceptable latency encoding to ns */
275 static u32 calc_l0s_acceptable(u32 encoding)
279 return (64 << encoding);
282 /* Convert L1 latency encoding to ns */
283 static u32 calc_l1_latency(u32 encoding)
286 return (65 * 1000); /* > 64us */
287 return (1000 << encoding);
290 /* Convert L1 acceptable latency encoding to ns */
291 static u32 calc_l1_acceptable(u32 encoding)
295 return (1000 << encoding);
298 struct aspm_register_info {
301 u32 latency_encoding_l0s;
302 u32 latency_encoding_l1;
305 static void pcie_get_aspm_reg(struct pci_dev *pdev,
306 struct aspm_register_info *info)
312 pos = pci_pcie_cap(pdev);
313 pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
314 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
315 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
316 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
317 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
318 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
321 static void pcie_aspm_check_latency(struct pci_dev *endpoint)
323 u32 latency, l1_switch_latency = 0;
324 struct aspm_latency *acceptable;
325 struct pcie_link_state *link;
327 /* Device not in D0 doesn't need latency check */
328 if ((endpoint->current_state != PCI_D0) &&
329 (endpoint->current_state != PCI_UNKNOWN))
332 link = endpoint->bus->self->link_state;
333 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
336 /* Check upstream direction L0s latency */
337 if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
338 (link->latency_up.l0s > acceptable->l0s))
339 link->aspm_capable &= ~ASPM_STATE_L0S_UP;
341 /* Check downstream direction L0s latency */
342 if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
343 (link->latency_dw.l0s > acceptable->l0s))
344 link->aspm_capable &= ~ASPM_STATE_L0S_DW;
347 * Every switch on the path to root complex need 1
348 * more microsecond for L1. Spec doesn't mention L0s.
350 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
351 if ((link->aspm_capable & ASPM_STATE_L1) &&
352 (latency + l1_switch_latency > acceptable->l1))
353 link->aspm_capable &= ~ASPM_STATE_L1;
354 l1_switch_latency += 1000;
360 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
362 struct pci_dev *child, *parent = link->pdev;
363 struct pci_bus *linkbus = parent->subordinate;
364 struct aspm_register_info upreg, dwreg;
367 /* Set enabled/disable so that we will disable ASPM later */
368 link->aspm_enabled = ASPM_STATE_ALL;
369 link->aspm_disable = ASPM_STATE_ALL;
373 /* Configure common clock before checking latencies */
374 pcie_aspm_configure_common_clock(link);
376 /* Get upstream/downstream components' register state */
377 pcie_get_aspm_reg(parent, &upreg);
378 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
379 pcie_get_aspm_reg(child, &dwreg);
384 * Note that we must not enable L0s in either direction on a
385 * given link unless components on both sides of the link each
388 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
389 link->aspm_support |= ASPM_STATE_L0S;
390 if (dwreg.enabled & PCIE_LINK_STATE_L0S)
391 link->aspm_enabled |= ASPM_STATE_L0S_UP;
392 if (upreg.enabled & PCIE_LINK_STATE_L0S)
393 link->aspm_enabled |= ASPM_STATE_L0S_DW;
394 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
395 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
398 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
399 link->aspm_support |= ASPM_STATE_L1;
400 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
401 link->aspm_enabled |= ASPM_STATE_L1;
402 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
403 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
405 /* Save default state */
406 link->aspm_default = link->aspm_enabled;
408 /* Setup initial capable state. Will be updated later */
409 link->aspm_capable = link->aspm_support;
411 * If the downstream component has pci bridge function, don't
414 list_for_each_entry(child, &linkbus->devices, bus_list) {
415 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
416 link->aspm_disable = ASPM_STATE_ALL;
421 /* Get and check endpoint acceptable latencies */
422 list_for_each_entry(child, &linkbus->devices, bus_list) {
425 struct aspm_latency *acceptable =
426 &link->acceptable[PCI_FUNC(child->devfn)];
428 if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
429 child->pcie_type != PCI_EXP_TYPE_LEG_END)
432 pos = pci_pcie_cap(child);
433 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32);
434 /* Calculate endpoint L0s acceptable latency */
435 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
436 acceptable->l0s = calc_l0s_acceptable(encoding);
437 /* Calculate endpoint L1 acceptable latency */
438 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
439 acceptable->l1 = calc_l1_acceptable(encoding);
441 pcie_aspm_check_latency(child);
445 static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
448 int pos = pci_pcie_cap(pdev);
450 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
453 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
456 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
458 u32 upstream = 0, dwstream = 0;
459 struct pci_dev *child, *parent = link->pdev;
460 struct pci_bus *linkbus = parent->subordinate;
462 /* Nothing to do if the link is already in the requested state */
463 state &= (link->aspm_capable & ~link->aspm_disable);
464 if (link->aspm_enabled == state)
466 /* Convert ASPM state to upstream/downstream ASPM register state */
467 if (state & ASPM_STATE_L0S_UP)
468 dwstream |= PCIE_LINK_STATE_L0S;
469 if (state & ASPM_STATE_L0S_DW)
470 upstream |= PCIE_LINK_STATE_L0S;
471 if (state & ASPM_STATE_L1) {
472 upstream |= PCIE_LINK_STATE_L1;
473 dwstream |= PCIE_LINK_STATE_L1;
476 * Spec 2.0 suggests all functions should be configured the
477 * same setting for ASPM. Enabling ASPM L1 should be done in
478 * upstream component first and then downstream, and vice
479 * versa for disabling ASPM L1. Spec doesn't mention L0S.
481 if (state & ASPM_STATE_L1)
482 pcie_config_aspm_dev(parent, upstream);
483 list_for_each_entry(child, &linkbus->devices, bus_list)
484 pcie_config_aspm_dev(child, dwstream);
485 if (!(state & ASPM_STATE_L1))
486 pcie_config_aspm_dev(parent, upstream);
488 link->aspm_enabled = state;
491 static void pcie_config_aspm_path(struct pcie_link_state *link)
494 pcie_config_aspm_link(link, policy_to_aspm_state(link));
499 static void free_link_state(struct pcie_link_state *link)
501 link->pdev->link_state = NULL;
505 static int pcie_aspm_sanity_check(struct pci_dev *pdev)
507 struct pci_dev *child;
515 * Some functions in a slot might not all be PCIe functions,
516 * very strange. Disable ASPM for the whole slot
518 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
519 pos = pci_pcie_cap(child);
523 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
524 * RBER bit to determine if a function is 1.1 version device
526 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32);
527 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
528 dev_printk(KERN_INFO, &child->dev, "disabling ASPM"
529 " on pre-1.1 PCIe device. You can enable it"
530 " with 'pcie_aspm=force'\n");
537 static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
539 struct pcie_link_state *link;
541 link = kzalloc(sizeof(*link), GFP_KERNEL);
544 INIT_LIST_HEAD(&link->sibling);
545 INIT_LIST_HEAD(&link->children);
546 INIT_LIST_HEAD(&link->link);
548 if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
549 struct pcie_link_state *parent;
550 parent = pdev->bus->parent->self->link_state;
555 link->parent = parent;
556 list_add(&link->link, &parent->children);
558 /* Setup a pointer to the root port link */
562 link->root = link->parent->root;
564 list_add(&link->sibling, &link_list);
565 pdev->link_state = link;
570 * pcie_aspm_init_link_state: Initiate PCI express link state.
571 * It is called after the pcie and its children devices are scaned.
572 * @pdev: the root port or switch downstream port
574 void pcie_aspm_init_link_state(struct pci_dev *pdev)
576 struct pcie_link_state *link;
577 int blacklist = !!pcie_aspm_sanity_check(pdev);
579 if (!pci_is_pcie(pdev) || pdev->link_state)
581 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
582 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
585 /* VIA has a strange chipset, root port is under a bridge */
586 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
590 down_read(&pci_bus_sem);
591 if (list_empty(&pdev->subordinate->devices))
594 mutex_lock(&aspm_lock);
595 link = alloc_pcie_link_state(pdev);
599 * Setup initial ASPM state. Note that we need to configure
600 * upstream links also because capable state of them can be
601 * update through pcie_aspm_cap_init().
603 pcie_aspm_cap_init(link, blacklist);
605 /* Setup initial Clock PM state */
606 pcie_clkpm_cap_init(link, blacklist);
609 * At this stage drivers haven't had an opportunity to change the
610 * link policy setting. Enabling ASPM on broken hardware can cripple
611 * it even before the driver has had a chance to disable ASPM, so
612 * default to a safe level right now. If we're enabling ASPM beyond
613 * the BIOS's expectation, we'll do so once pci_enable_device() is
616 if (aspm_policy != POLICY_POWERSAVE) {
617 pcie_config_aspm_path(link);
618 pcie_set_clkpm(link, policy_to_clkpm_state(link));
622 mutex_unlock(&aspm_lock);
624 up_read(&pci_bus_sem);
627 /* Recheck latencies and update aspm_capable for links under the root */
628 static void pcie_update_aspm_capable(struct pcie_link_state *root)
630 struct pcie_link_state *link;
631 BUG_ON(root->parent);
632 list_for_each_entry(link, &link_list, sibling) {
633 if (link->root != root)
635 link->aspm_capable = link->aspm_support;
637 list_for_each_entry(link, &link_list, sibling) {
638 struct pci_dev *child;
639 struct pci_bus *linkbus = link->pdev->subordinate;
640 if (link->root != root)
642 list_for_each_entry(child, &linkbus->devices, bus_list) {
643 if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT) &&
644 (child->pcie_type != PCI_EXP_TYPE_LEG_END))
646 pcie_aspm_check_latency(child);
651 /* @pdev: the endpoint device */
652 void pcie_aspm_exit_link_state(struct pci_dev *pdev)
654 struct pci_dev *parent = pdev->bus->self;
655 struct pcie_link_state *link, *root, *parent_link;
657 if (!pci_is_pcie(pdev) || !parent || !parent->link_state)
659 if ((parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
660 (parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
663 down_read(&pci_bus_sem);
664 mutex_lock(&aspm_lock);
666 * All PCIe functions are in one slot, remove one function will remove
667 * the whole slot, so just wait until we are the last function left.
669 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices))
672 link = parent->link_state;
674 parent_link = link->parent;
676 /* All functions are removed, so just disable ASPM for the link */
677 pcie_config_aspm_link(link, 0);
678 list_del(&link->sibling);
679 list_del(&link->link);
680 /* Clock PM is for endpoint device */
681 free_link_state(link);
683 /* Recheck latencies and configure upstream links */
685 pcie_update_aspm_capable(root);
686 pcie_config_aspm_path(parent_link);
689 mutex_unlock(&aspm_lock);
690 up_read(&pci_bus_sem);
693 /* @pdev: the root port or switch downstream port */
694 void pcie_aspm_pm_state_change(struct pci_dev *pdev)
696 struct pcie_link_state *link = pdev->link_state;
698 if (aspm_disabled || !pci_is_pcie(pdev) || !link)
700 if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
701 (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
704 * Devices changed PM state, we should recheck if latency
705 * meets all functions' requirement
707 down_read(&pci_bus_sem);
708 mutex_lock(&aspm_lock);
709 pcie_update_aspm_capable(link->root);
710 pcie_config_aspm_path(link);
711 mutex_unlock(&aspm_lock);
712 up_read(&pci_bus_sem);
715 void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
717 struct pcie_link_state *link = pdev->link_state;
719 if (aspm_disabled || !pci_is_pcie(pdev) || !link)
722 if (aspm_policy != POLICY_POWERSAVE)
725 if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
726 (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
729 down_read(&pci_bus_sem);
730 mutex_lock(&aspm_lock);
731 pcie_config_aspm_path(link);
732 pcie_set_clkpm(link, policy_to_clkpm_state(link));
733 mutex_unlock(&aspm_lock);
734 up_read(&pci_bus_sem);
738 * pci_disable_link_state - disable pci device's link state, so the link will
739 * never enter specific states
741 static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem,
744 struct pci_dev *parent = pdev->bus->self;
745 struct pcie_link_state *link;
747 if (aspm_disabled && !force)
750 if (!pci_is_pcie(pdev))
753 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
754 pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
756 if (!parent || !parent->link_state)
760 down_read(&pci_bus_sem);
761 mutex_lock(&aspm_lock);
762 link = parent->link_state;
763 if (state & PCIE_LINK_STATE_L0S)
764 link->aspm_disable |= ASPM_STATE_L0S;
765 if (state & PCIE_LINK_STATE_L1)
766 link->aspm_disable |= ASPM_STATE_L1;
767 pcie_config_aspm_link(link, policy_to_aspm_state(link));
769 if (state & PCIE_LINK_STATE_CLKPM) {
770 link->clkpm_capable = 0;
771 pcie_set_clkpm(link, 0);
773 mutex_unlock(&aspm_lock);
775 up_read(&pci_bus_sem);
778 void pci_disable_link_state_locked(struct pci_dev *pdev, int state)
780 __pci_disable_link_state(pdev, state, false, false);
782 EXPORT_SYMBOL(pci_disable_link_state_locked);
784 void pci_disable_link_state(struct pci_dev *pdev, int state)
786 __pci_disable_link_state(pdev, state, true, false);
788 EXPORT_SYMBOL(pci_disable_link_state);
790 void pcie_clear_aspm(struct pci_bus *bus)
792 struct pci_dev *child;
795 * Clear any ASPM setup that the firmware has carried out on this bus
797 list_for_each_entry(child, &bus->devices, bus_list) {
798 __pci_disable_link_state(child, PCIE_LINK_STATE_L0S |
800 PCIE_LINK_STATE_CLKPM,
805 static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
808 struct pcie_link_state *link;
812 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
813 if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
815 if (i >= ARRAY_SIZE(policy_str))
817 if (i == aspm_policy)
820 down_read(&pci_bus_sem);
821 mutex_lock(&aspm_lock);
823 list_for_each_entry(link, &link_list, sibling) {
824 pcie_config_aspm_link(link, policy_to_aspm_state(link));
825 pcie_set_clkpm(link, policy_to_clkpm_state(link));
827 mutex_unlock(&aspm_lock);
828 up_read(&pci_bus_sem);
832 static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
835 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
836 if (i == aspm_policy)
837 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
839 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
843 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
846 #ifdef CONFIG_PCIEASPM_DEBUG
847 static ssize_t link_state_show(struct device *dev,
848 struct device_attribute *attr,
851 struct pci_dev *pci_device = to_pci_dev(dev);
852 struct pcie_link_state *link_state = pci_device->link_state;
854 return sprintf(buf, "%d\n", link_state->aspm_enabled);
857 static ssize_t link_state_store(struct device *dev,
858 struct device_attribute *attr,
862 struct pci_dev *pdev = to_pci_dev(dev);
863 struct pcie_link_state *link, *root = pdev->link_state->root;
864 u32 val = buf[0] - '0', state = 0;
868 if (n < 1 || val > 3)
871 /* Convert requested state to ASPM state */
872 if (val & PCIE_LINK_STATE_L0S)
873 state |= ASPM_STATE_L0S;
874 if (val & PCIE_LINK_STATE_L1)
875 state |= ASPM_STATE_L1;
877 down_read(&pci_bus_sem);
878 mutex_lock(&aspm_lock);
879 list_for_each_entry(link, &link_list, sibling) {
880 if (link->root != root)
882 pcie_config_aspm_link(link, state);
884 mutex_unlock(&aspm_lock);
885 up_read(&pci_bus_sem);
889 static ssize_t clk_ctl_show(struct device *dev,
890 struct device_attribute *attr,
893 struct pci_dev *pci_device = to_pci_dev(dev);
894 struct pcie_link_state *link_state = pci_device->link_state;
896 return sprintf(buf, "%d\n", link_state->clkpm_enabled);
899 static ssize_t clk_ctl_store(struct device *dev,
900 struct device_attribute *attr,
904 struct pci_dev *pdev = to_pci_dev(dev);
911 down_read(&pci_bus_sem);
912 mutex_lock(&aspm_lock);
913 pcie_set_clkpm_nocheck(pdev->link_state, !!state);
914 mutex_unlock(&aspm_lock);
915 up_read(&pci_bus_sem);
920 static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
921 static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
923 static char power_group[] = "power";
924 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
926 struct pcie_link_state *link_state = pdev->link_state;
928 if (!pci_is_pcie(pdev) ||
929 (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
930 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
933 if (link_state->aspm_support)
934 sysfs_add_file_to_group(&pdev->dev.kobj,
935 &dev_attr_link_state.attr, power_group);
936 if (link_state->clkpm_capable)
937 sysfs_add_file_to_group(&pdev->dev.kobj,
938 &dev_attr_clk_ctl.attr, power_group);
941 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
943 struct pcie_link_state *link_state = pdev->link_state;
945 if (!pci_is_pcie(pdev) ||
946 (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
947 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
950 if (link_state->aspm_support)
951 sysfs_remove_file_from_group(&pdev->dev.kobj,
952 &dev_attr_link_state.attr, power_group);
953 if (link_state->clkpm_capable)
954 sysfs_remove_file_from_group(&pdev->dev.kobj,
955 &dev_attr_clk_ctl.attr, power_group);
959 static int __init pcie_aspm_disable(char *str)
961 if (!strcmp(str, "off")) {
962 aspm_policy = POLICY_DEFAULT;
964 aspm_support_enabled = false;
965 printk(KERN_INFO "PCIe ASPM is disabled\n");
966 } else if (!strcmp(str, "force")) {
968 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
973 __setup("pcie_aspm=", pcie_aspm_disable);
975 void pcie_no_aspm(void)
978 * Disabling ASPM is intended to prevent the kernel from modifying
979 * existing hardware state, not to clear existing state. To that end:
980 * (a) set policy to POLICY_DEFAULT in order to avoid changing state
981 * (b) prevent userspace from changing policy
984 aspm_policy = POLICY_DEFAULT;
990 * pcie_aspm_enabled - is PCIe ASPM enabled?
992 * Returns true if ASPM has not been disabled by the command-line option
995 int pcie_aspm_enabled(void)
997 return !aspm_disabled;
999 EXPORT_SYMBOL(pcie_aspm_enabled);
1001 bool pcie_aspm_support_enabled(void)
1003 return aspm_support_enabled;
1005 EXPORT_SYMBOL(pcie_aspm_support_enabled);