diff --git a/drivers/cpuidle/cpuidle-haltpoll.c b/drivers/cpuidle/cpuidle-haltpoll.c
index 926a6bcfed9bd4c133c706777819de8ee47f1679..fc351f093ce1afef947a4b164fdb0e0b91d28448 100644
--- a/drivers/cpuidle/cpuidle-haltpoll.c
+++ b/drivers/cpuidle/cpuidle-haltpoll.c
@@ -22,7 +22,7 @@ static bool force __read_mostly;
module_param(force, bool, 0444);
MODULE_PARM_DESC(force, "Load unconditionally");
-static struct cpuidle_device __percpu *haltpoll_cpuidle_devices;
+static struct cpuidle_device_wrapper __percpu *haltpoll_cpuidle_dev_wrap;
static enum cpuhp_state haltpoll_hp_state;
static int default_enter_idle(struct cpuidle_device *dev,
@@ -36,29 +36,34 @@ static int default_enter_idle(struct cpuidle_device *dev,
return index;
}
-static struct cpuidle_driver haltpoll_driver = {
- .name = "haltpoll",
- .governor = "haltpoll",
- .states = {
- { /* entry 0 is for polling */ },
- {
- .enter = default_enter_idle,
- .exit_latency = 1,
- .target_residency = 1,
- .power_usage = -1,
- .name = "haltpoll idle",
- .desc = "default architecture idle",
+static struct cpuidle_driver_wrapper haltpoll_driver_wrapper = {
+ .drv = {
+ .name = "haltpoll",
+ .states = {
+ { /* entry 0 is for polling */ },
+ {
+ .enter = default_enter_idle,
+ .exit_latency = 1,
+ .target_residency = 1,
+ .power_usage = -1,
+ .name = "haltpoll idle",
+ .desc =
+ "default architecture idle",
+ },
},
+ .safe_state_index = 0,
+ .state_count = 2,
},
- .safe_state_index = 0,
- .state_count = 2,
+ .governor = "haltpoll",
};
static int haltpoll_cpu_online(unsigned int cpu)
{
struct cpuidle_device *dev;
+ struct cpuidle_device_wrapper *devw;
- dev = per_cpu_ptr(haltpoll_cpuidle_devices, cpu);
+ devw = per_cpu_ptr(haltpoll_cpuidle_dev_wrap, cpu);
+ dev = &(devw->dev);
if (!dev->registered) {
dev->cpu = cpu;
if (cpuidle_register_device(dev)) {
@@ -74,8 +79,10 @@ static int haltpoll_cpu_online(unsigned int cpu)
static int haltpoll_cpu_offline(unsigned int cpu)
{
struct cpuidle_device *dev;
+ struct cpuidle_device_wrapper *devw;
- dev = per_cpu_ptr(haltpoll_cpuidle_devices, cpu);
+ devw = per_cpu_ptr(haltpoll_cpuidle_dev_wrap, cpu);
+ dev = &(devw->dev);
if (dev->registered) {
arch_haltpoll_disable(cpu);
cpuidle_unregister_device(dev);
@@ -88,10 +95,10 @@ static void haltpoll_uninit(void)
{
if (haltpoll_hp_state)
cpuhp_remove_state(haltpoll_hp_state);
- cpuidle_unregister_driver(&haltpoll_driver);
+ cpuidle_unregister_driver(&(haltpoll_driver_wrapper.drv));
- free_percpu(haltpoll_cpuidle_devices);
- haltpoll_cpuidle_devices = NULL;
+ free_percpu(haltpoll_cpuidle_dev_wrap);
+ haltpoll_cpuidle_dev_wrap = NULL;
}
static bool haltpoll_want(void)
@@ -102,7 +109,7 @@ static bool haltpoll_want(void)
static int __init haltpoll_init(void)
{
int ret;
- struct cpuidle_driver *drv = &haltpoll_driver;
+ struct cpuidle_driver *drv = &(haltpoll_driver_wrapper.drv);
cpuidle_poll_state_init(drv);
@@ -113,8 +120,8 @@ static int __init haltpoll_init(void)
if (ret < 0)
return ret;
- haltpoll_cpuidle_devices = alloc_percpu(struct cpuidle_device);
- if (haltpoll_cpuidle_devices == NULL) {
+ haltpoll_cpuidle_dev_wrap = alloc_percpu(struct cpuidle_device_wrapper);
+ if (haltpoll_cpuidle_dev_wrap == NULL) {
cpuidle_unregister_driver(drv);
return -ENOMEM;
}
diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c
index 55eb3d152521b9cf32b248eca74eb2b952972219..b7a7125f1cded5113d2ae5ece55d2897a8985f26 100644
--- a/drivers/cpuidle/cpuidle.c
+++ b/drivers/cpuidle/cpuidle.c
@@ -330,11 +330,13 @@ void cpuidle_reflect(struct cpuidle_device *dev, int index)
u64 cpuidle_poll_time(struct cpuidle_driver *drv,
struct cpuidle_device *dev)
{
+ struct cpuidle_device_wrapper *devw =
+ container_of(dev, struct cpuidle_device_wrapper, dev);
int i;
u64 limit_ns;
- if (dev->poll_limit_ns)
- return dev->poll_limit_ns;
+ if (devw->poll_limit_ns)
+ return devw->poll_limit_ns;
limit_ns = TICK_NSEC;
for (i = 1; i < drv->state_count; i++) {
@@ -346,9 +348,9 @@ u64 cpuidle_poll_time(struct cpuidle_driver *drv,
break;
}
- dev->poll_limit_ns = limit_ns;
+ devw->poll_limit_ns = limit_ns;
- return dev->poll_limit_ns;
+ return devw->poll_limit_ns;
}
/**
diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c
index 9db154224999c9e6d7a2704b0b685bdb7658b134..47930a0ecb0f14cbbf8bbbc983ea1a49e66e9257 100644
--- a/drivers/cpuidle/driver.c
+++ b/drivers/cpuidle/driver.c
@@ -254,16 +254,18 @@ static void __cpuidle_unregister_driver(struct cpuidle_driver *drv)
int cpuidle_register_driver(struct cpuidle_driver *drv)
{
struct cpuidle_governor *gov;
+ struct cpuidle_driver_wrapper *drvw;
int ret;
spin_lock(&cpuidle_driver_lock);
ret = __cpuidle_register_driver(drv);
spin_unlock(&cpuidle_driver_lock);
+ drvw = container_of(drv, struct cpuidle_driver_wrapper, drv);
- if (!ret && !strlen(param_governor) && drv->governor &&
+ if (!ret && !strlen(param_governor) && drvw->governor &&
(cpuidle_get_driver() == drv)) {
mutex_lock(&cpuidle_lock);
- gov = cpuidle_find_governor(drv->governor);
+ gov = cpuidle_find_governor(drvw->governor);
if (gov) {
cpuidle_prev_governor = cpuidle_curr_governor;
if (cpuidle_switch_governor(gov) < 0)
diff --git a/drivers/cpuidle/governors/haltpoll.c b/drivers/cpuidle/governors/haltpoll.c
index ce550ab98e66595357ff7cd44685f85bee5859c9..3490ba7de14151d8a9ca0609889e4676589c261e 100644
--- a/drivers/cpuidle/governors/haltpoll.c
+++ b/drivers/cpuidle/governors/haltpoll.c
@@ -53,6 +53,8 @@ static int haltpoll_select(struct cpuidle_driver *drv,
struct cpuidle_device *dev,
bool *stop_tick)
{
+ struct cpuidle_device_wrapper *devw =
+ container_of(dev, struct cpuidle_device_wrapper, dev);
int latency_req = cpuidle_governor_latency_req(dev->cpu);
if (!drv->state_count || latency_req == 0) {
@@ -60,11 +62,11 @@ static int haltpoll_select(struct cpuidle_driver *drv,
return 0;
}
- if (dev->poll_limit_ns == 0)
+ if (devw->poll_limit_ns == 0)
return 1;
/* Last state was poll? */
- if (dev->last_state_idx == 0) {
+ if (devw->last_state_idx == 0) {
/* Halt if no event occurred on poll window */
if (dev->poll_time_limit == true)
return 1;
@@ -81,31 +83,33 @@ static int haltpoll_select(struct cpuidle_driver *drv,
static void adjust_poll_limit(struct cpuidle_device *dev, unsigned int block_us)
{
+ struct cpuidle_device_wrapper *devw =
+ container_of(dev, struct cpuidle_device_wrapper, dev);
unsigned int val;
u64 block_ns = block_us*NSEC_PER_USEC;
/* Grow cpu_halt_poll_us if
* cpu_halt_poll_us < block_ns < guest_halt_poll_us
*/
- if (block_ns > dev->poll_limit_ns && block_ns <= guest_halt_poll_ns) {
- val = dev->poll_limit_ns * guest_halt_poll_grow;
+ if (block_ns > devw->poll_limit_ns && block_ns <= guest_halt_poll_ns) {
+ val = devw->poll_limit_ns * guest_halt_poll_grow;
if (val < guest_halt_poll_grow_start)
val = guest_halt_poll_grow_start;
if (val > guest_halt_poll_ns)
val = guest_halt_poll_ns;
- dev->poll_limit_ns = val;
+ devw->poll_limit_ns = val;
} else if (block_ns > guest_halt_poll_ns &&
guest_halt_poll_allow_shrink) {
unsigned int shrink = guest_halt_poll_shrink;
- val = dev->poll_limit_ns;
+ val = devw->poll_limit_ns;
if (shrink == 0)
val = 0;
else
val /= shrink;
- dev->poll_limit_ns = val;
+ devw->poll_limit_ns = val;
}
}
@@ -116,7 +120,9 @@ static void adjust_poll_limit(struct cpuidle_device *dev, unsigned int block_us)
*/
static void haltpoll_reflect(struct cpuidle_device *dev, int index)
{
- dev->last_state_idx = index;
+ struct cpuidle_device_wrapper *devw =
+ container_of(dev, struct cpuidle_device_wrapper, dev);
+ devw->last_state_idx = index;
if (index != 0)
adjust_poll_limit(dev, dev->last_residency);
@@ -130,7 +136,10 @@ static void haltpoll_reflect(struct cpuidle_device *dev, int index)
static int haltpoll_enable_device(struct cpuidle_driver *drv,
struct cpuidle_device *dev)
{
- dev->poll_limit_ns = 0;
+ struct cpuidle_device_wrapper *devw =
+ container_of(dev, struct cpuidle_device_wrapper, dev);
+
+ devw->poll_limit_ns = 0;
return 0;
}
diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c
index ecf858e97648eb056802d3080f88a8f6859d24ce..704880a6612a76341b4f6746fd109366c940e41d 100644
--- a/drivers/cpuidle/governors/ladder.c
+++ b/drivers/cpuidle/governors/ladder.c
@@ -38,6 +38,7 @@ struct ladder_device_state {
struct ladder_device {
struct ladder_device_state states[CPUIDLE_STATE_MAX];
+ int last_state_idx;
};
static DEFINE_PER_CPU(struct ladder_device, ladder_devices);
@@ -48,13 +49,12 @@ static DEFINE_PER_CPU(struct ladder_device, ladder_devices);
* @old_idx: the current state index
* @new_idx: the new target state index
*/
-static inline void ladder_do_selection(struct cpuidle_device *dev,
- struct ladder_device *ldev,
+static inline void ladder_do_selection(struct ladder_device *ldev,
int old_idx, int new_idx)
{
ldev->states[old_idx].stats.promotion_count = 0;
ldev->states[old_idx].stats.demotion_count = 0;
- dev->last_state_idx = new_idx;
+ ldev->last_state_idx = new_idx;
}
/**
@@ -68,13 +68,13 @@ static int ladder_select_state(struct cpuidle_driver *drv,
{
struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
struct ladder_device_state *last_state;
- int last_residency, last_idx = dev->last_state_idx;
+ int last_residency, last_idx = ldev->last_state_idx;
int first_idx = drv->states[0].flags & CPUIDLE_FLAG_POLLING ? 1 : 0;
int latency_req = cpuidle_governor_latency_req(dev->cpu);
/* Special case when user has set very strict latency requirement */
if (unlikely(latency_req == 0)) {
- ladder_do_selection(dev, ldev, last_idx, 0);
+ ladder_do_selection(ldev, last_idx, 0);
return 0;
}
@@ -91,7 +91,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
last_state->stats.promotion_count++;
last_state->stats.demotion_count = 0;
if (last_state->stats.promotion_count >= last_state->threshold.promotion_count) {
- ladder_do_selection(dev, ldev, last_idx, last_idx + 1);
+ ladder_do_selection(ldev, last_idx, last_idx + 1);
return last_idx + 1;
}
}
@@ -107,7 +107,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
if (drv->states[i].exit_latency <= latency_req)
break;
}
- ladder_do_selection(dev, ldev, last_idx, i);
+ ladder_do_selection(ldev, last_idx, i);
return i;
}
@@ -116,7 +116,7 @@ static int ladder_select_state(struct cpuidle_driver *drv,
last_state->stats.demotion_count++;
last_state->stats.promotion_count = 0;
if (last_state->stats.demotion_count >= last_state->threshold.demotion_count) {
- ladder_do_selection(dev, ldev, last_idx, last_idx - 1);
+ ladder_do_selection(ldev, last_idx, last_idx - 1);
return last_idx - 1;
}
}
@@ -139,7 +139,7 @@ static int ladder_enable_device(struct cpuidle_driver *drv,
struct ladder_device_state *lstate;
struct cpuidle_state *state;
- dev->last_state_idx = first_idx;
+ ldev->last_state_idx = first_idx;
for (i = first_idx; i < drv->state_count; i++) {
state = &drv->states[i];
@@ -167,8 +167,9 @@ static int ladder_enable_device(struct cpuidle_driver *drv,
*/
static void ladder_reflect(struct cpuidle_device *dev, int index)
{
+ struct ladder_device *ldev = this_cpu_ptr(&ladder_devices);
if (index > 0)
- dev->last_state_idx = index;
+ ldev->last_state_idx = index;
}
static struct cpuidle_governor ladder_governor = {
diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c
index c447014d1d727a73bd4ccb59c4e1b760ede21a89..3a48e4e1c175db48e59813441ca6841281d8c5ec 100644
--- a/drivers/cpuidle/governors/menu.c
+++ b/drivers/cpuidle/governors/menu.c
@@ -119,6 +119,7 @@
*/
struct menu_device {
+ int last_state_idx;
int needs_update;
int tick_wakeup;
@@ -464,7 +465,7 @@ static void menu_reflect(struct cpuidle_device *dev, int index)
{
struct menu_device *data = this_cpu_ptr(&menu_devices);
- dev->last_state_idx = index;
+ data->last_state_idx = index;
data->needs_update = 1;
data->tick_wakeup = tick_nohz_idle_got_tick();
}
@@ -477,7 +478,7 @@ static void menu_reflect(struct cpuidle_device *dev, int index)
static void menu_update(struct cpuidle_driver *drv, struct cpuidle_device *dev)
{
struct menu_device *data = this_cpu_ptr(&menu_devices);
- int last_idx = dev->last_state_idx;
+ int last_idx = data->last_state_idx;
struct cpuidle_state *target = &drv->states[last_idx];
unsigned int measured_us;
unsigned int new_factor;
diff --git a/drivers/cpuidle/sysfs.c b/drivers/cpuidle/sysfs.c
index 6799a938f536d6c32ea9054e0e5cd55d2a2b4541..f966a343daa71433b0e54c883dadd9a4f9e3d22e 100644
--- a/drivers/cpuidle/sysfs.c
+++ b/drivers/cpuidle/sysfs.c
@@ -411,12 +411,14 @@ static ssize_t cpuidle_state_store(struct kobject *kobj, struct attribute *attr,
struct cpuidle_state_usage *state_usage = kobj_to_state_usage(kobj);
struct cpuidle_state_attr *cattr = attr_to_stateattr(attr);
struct cpuidle_device *dev = kobj_to_device(kobj);
+ struct cpuidle_device_wrapper *devw =
+ container_of(dev, struct cpuidle_device_wrapper, dev);
if (cattr->store)
ret = cattr->store(state, state_usage, buf, size);
/* reset poll time cache */
- dev->poll_limit_ns = 0;
+ devw->poll_limit_ns = 0;
return ret;
}
diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h
index 43010919581ce495b8c4270160a62bfa1d45fac2..364e28be3155f5f9f9d9f1de15af373abac9b459 100644
--- a/include/linux/cpuidle.h
+++ b/include/linux/cpuidle.h
@@ -84,9 +84,7 @@ struct cpuidle_device {
unsigned int poll_time_limit:1;
unsigned int cpu;
- int last_state_idx;
int last_residency;
- u64 poll_limit_ns;
struct cpuidle_state_usage states_usage[CPUIDLE_STATE_MAX];
struct cpuidle_state_kobj *kobjs[CPUIDLE_STATE_MAX];
struct cpuidle_driver_kobj *kobj_driver;
@@ -99,6 +97,12 @@ struct cpuidle_device {
#endif
};
+struct cpuidle_device_wrapper {
+ struct cpuidle_device dev;
+ int last_state_idx;
+ u64 poll_limit_ns;
+};
+
DECLARE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
DECLARE_PER_CPU(struct cpuidle_device, cpuidle_dev);
@@ -130,9 +134,12 @@ struct cpuidle_driver {
/* the driver handles the cpus in cpumask */
struct cpumask *cpumask;
+};
+struct cpuidle_driver_wrapper {
+ struct cpuidle_driver drv;
/* preferred governor to switch at register time */
- const char *governor;
+ const char *governor;
};
#ifdef CONFIG_CPU_IDLE