From a7fe747a40ae35f67c4557096409a5e56fbcec88 Mon Sep 17 00:00:00 2001 From: Luca Abeni Date: Sun, 6 Aug 2017 23:54:45 +0200 Subject: [PATCH 2/2] sched/deadline: remove migration-related code --- include/linux/sched.h | 1 - kernel/sched/Makefile | 2 +- kernel/sched/core.c | 2 - kernel/sched/cpudeadline.c | 278 ------------- kernel/sched/cpudeadline.h | 34 -- kernel/sched/deadline.c | 943 +-------------------------------------------- kernel/sched/debug.c | 1 - kernel/sched/sched.h | 32 -- kernel/sched/topology.c | 15 +- 9 files changed, 4 insertions(+), 1304 deletions(-) delete mode 100644 kernel/sched/cpudeadline.c delete mode 100644 kernel/sched/cpudeadline.h diff --git a/include/linux/sched.h b/include/linux/sched.h index 8c96518fc94f..3462afd04f2e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -599,7 +599,6 @@ struct task_struct { struct list_head tasks; #ifdef CONFIG_SMP struct plist_node pushable_tasks; - struct rb_node pushable_dl_tasks; #endif struct mm_struct *mm; diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 53f0164ed362..d45fdd0d18b2 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -18,7 +18,7 @@ endif obj-y += core.o loadavg.o clock.o cputime.o obj-y += idle_task.o fair.o rt.o deadline.o obj-y += wait.o wait_bit.o swait.o completion.o idle.o -obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o topology.o stop_task.o +obj-$(CONFIG_SMP) += cpupri.o topology.o stop_task.o obj-$(CONFIG_SCHED_AUTOGROUP) += autogroup.o obj-$(CONFIG_SCHEDSTATS) += stats.o obj-$(CONFIG_SCHED_DEBUG) += debug.o diff --git a/kernel/sched/core.c b/kernel/sched/core.c index a8a8153123d5..4b25021d3537 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2398,7 +2398,6 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) init_task_preempt_count(p); #ifdef CONFIG_SMP plist_node_init(&p->pushable_tasks, MAX_PRIO); - RB_CLEAR_NODE(&p->pushable_dl_tasks); #endif put_cpu(); @@ -5684,7 +5683,6 @@ void __init sched_init_smp(void) free_cpumask_var(non_isolated_cpus); init_sched_rt_class(); - init_sched_dl_class(); sched_init_smt(); diff --git a/kernel/sched/cpudeadline.c b/kernel/sched/cpudeadline.c deleted file mode 100644 index fba235c7d026..000000000000 --- a/kernel/sched/cpudeadline.c +++ /dev/null @@ -1,278 +0,0 @@ -/* - * kernel/sched/cpudl.c - * - * Global CPU deadline management - * - * Author: Juri Lelli - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; version 2 - * of the License. - */ - -#include -#include -#include -#include "cpudeadline.h" - -static inline int parent(int i) -{ - return (i - 1) >> 1; -} - -static inline int left_child(int i) -{ - return (i << 1) + 1; -} - -static inline int right_child(int i) -{ - return (i << 1) + 2; -} - -static void cpudl_heapify_down(struct cpudl *cp, int idx) -{ - int l, r, largest; - - int orig_cpu = cp->elements[idx].cpu; - u64 orig_dl = cp->elements[idx].dl; - - if (left_child(idx) >= cp->size) - return; - - /* adapted from lib/prio_heap.c */ - while(1) { - u64 largest_dl; - l = left_child(idx); - r = right_child(idx); - largest = idx; - largest_dl = orig_dl; - - if ((l < cp->size) && dl_time_before(orig_dl, - cp->elements[l].dl)) { - largest = l; - largest_dl = cp->elements[l].dl; - } - if ((r < cp->size) && dl_time_before(largest_dl, - cp->elements[r].dl)) - largest = r; - - if (largest == idx) - break; - - /* pull largest child onto idx */ - cp->elements[idx].cpu = cp->elements[largest].cpu; - cp->elements[idx].dl = cp->elements[largest].dl; - cp->elements[cp->elements[idx].cpu].idx = idx; - idx = largest; - } - /* actual push down of saved original values orig_* */ - cp->elements[idx].cpu = orig_cpu; - cp->elements[idx].dl = orig_dl; - cp->elements[cp->elements[idx].cpu].idx = idx; -} - -static void cpudl_heapify_up(struct cpudl *cp, int idx) -{ - int p; - - int orig_cpu = cp->elements[idx].cpu; - u64 orig_dl = cp->elements[idx].dl; - - if (idx == 0) - return; - - do { - p = parent(idx); - if (dl_time_before(orig_dl, cp->elements[p].dl)) - break; - /* pull parent onto idx */ - cp->elements[idx].cpu = cp->elements[p].cpu; - cp->elements[idx].dl = cp->elements[p].dl; - cp->elements[cp->elements[idx].cpu].idx = idx; - idx = p; - } while (idx != 0); - /* actual push up of saved original values orig_* */ - cp->elements[idx].cpu = orig_cpu; - cp->elements[idx].dl = orig_dl; - cp->elements[cp->elements[idx].cpu].idx = idx; -} - -static void cpudl_heapify(struct cpudl *cp, int idx) -{ - if (idx > 0 && dl_time_before(cp->elements[parent(idx)].dl, - cp->elements[idx].dl)) - cpudl_heapify_up(cp, idx); - else - cpudl_heapify_down(cp, idx); -} - -static inline int cpudl_maximum(struct cpudl *cp) -{ - return cp->elements[0].cpu; -} - -/* - * cpudl_find - find the best (later-dl) CPU in the system - * @cp: the cpudl max-heap context - * @p: the task - * @later_mask: a mask to fill in with the selected CPUs (or NULL) - * - * Returns: int - best CPU (heap maximum if suitable) - */ -int cpudl_find(struct cpudl *cp, struct task_struct *p, - struct cpumask *later_mask) -{ - int best_cpu = -1; - const struct sched_dl_entity *dl_se = &p->dl; - - if (later_mask && - cpumask_and(later_mask, cp->free_cpus, &p->cpus_allowed)) { - best_cpu = cpumask_any(later_mask); - goto out; - } else if (cpumask_test_cpu(cpudl_maximum(cp), &p->cpus_allowed) && - dl_time_before(dl_se->deadline, cp->elements[0].dl)) { - best_cpu = cpudl_maximum(cp); - if (later_mask) - cpumask_set_cpu(best_cpu, later_mask); - } - -out: - WARN_ON(best_cpu != -1 && !cpu_present(best_cpu)); - - return best_cpu; -} - -/* - * cpudl_clear - remove a cpu from the cpudl max-heap - * @cp: the cpudl max-heap context - * @cpu: the target cpu - * - * Notes: assumes cpu_rq(cpu)->lock is locked - * - * Returns: (void) - */ -void cpudl_clear(struct cpudl *cp, int cpu) -{ - int old_idx, new_cpu; - unsigned long flags; - - WARN_ON(!cpu_present(cpu)); - - raw_spin_lock_irqsave(&cp->lock, flags); - - old_idx = cp->elements[cpu].idx; - if (old_idx == IDX_INVALID) { - /* - * Nothing to remove if old_idx was invalid. - * This could happen if a rq_offline_dl is - * called for a CPU without -dl tasks running. - */ - } else { - new_cpu = cp->elements[cp->size - 1].cpu; - cp->elements[old_idx].dl = cp->elements[cp->size - 1].dl; - cp->elements[old_idx].cpu = new_cpu; - cp->size--; - cp->elements[new_cpu].idx = old_idx; - cp->elements[cpu].idx = IDX_INVALID; - cpudl_heapify(cp, old_idx); - - cpumask_set_cpu(cpu, cp->free_cpus); - } - raw_spin_unlock_irqrestore(&cp->lock, flags); -} - -/* - * cpudl_set - update the cpudl max-heap - * @cp: the cpudl max-heap context - * @cpu: the target cpu - * @dl: the new earliest deadline for this cpu - * - * Notes: assumes cpu_rq(cpu)->lock is locked - * - * Returns: (void) - */ -void cpudl_set(struct cpudl *cp, int cpu, u64 dl) -{ - int old_idx; - unsigned long flags; - - WARN_ON(!cpu_present(cpu)); - - raw_spin_lock_irqsave(&cp->lock, flags); - - old_idx = cp->elements[cpu].idx; - if (old_idx == IDX_INVALID) { - int new_idx = cp->size++; - cp->elements[new_idx].dl = dl; - cp->elements[new_idx].cpu = cpu; - cp->elements[cpu].idx = new_idx; - cpudl_heapify_up(cp, new_idx); - cpumask_clear_cpu(cpu, cp->free_cpus); - } else { - cp->elements[old_idx].dl = dl; - cpudl_heapify(cp, old_idx); - } - - raw_spin_unlock_irqrestore(&cp->lock, flags); -} - -/* - * cpudl_set_freecpu - Set the cpudl.free_cpus - * @cp: the cpudl max-heap context - * @cpu: rd attached cpu - */ -void cpudl_set_freecpu(struct cpudl *cp, int cpu) -{ - cpumask_set_cpu(cpu, cp->free_cpus); -} - -/* - * cpudl_clear_freecpu - Clear the cpudl.free_cpus - * @cp: the cpudl max-heap context - * @cpu: rd attached cpu - */ -void cpudl_clear_freecpu(struct cpudl *cp, int cpu) -{ - cpumask_clear_cpu(cpu, cp->free_cpus); -} - -/* - * cpudl_init - initialize the cpudl structure - * @cp: the cpudl max-heap context - */ -int cpudl_init(struct cpudl *cp) -{ - int i; - - memset(cp, 0, sizeof(*cp)); - raw_spin_lock_init(&cp->lock); - cp->size = 0; - - cp->elements = kcalloc(nr_cpu_ids, - sizeof(struct cpudl_item), - GFP_KERNEL); - if (!cp->elements) - return -ENOMEM; - - if (!zalloc_cpumask_var(&cp->free_cpus, GFP_KERNEL)) { - kfree(cp->elements); - return -ENOMEM; - } - - for_each_possible_cpu(i) - cp->elements[i].idx = IDX_INVALID; - - return 0; -} - -/* - * cpudl_cleanup - clean up the cpudl structure - * @cp: the cpudl max-heap context - */ -void cpudl_cleanup(struct cpudl *cp) -{ - free_cpumask_var(cp->free_cpus); - kfree(cp->elements); -} diff --git a/kernel/sched/cpudeadline.h b/kernel/sched/cpudeadline.h deleted file mode 100644 index f7da8c55bba0..000000000000 --- a/kernel/sched/cpudeadline.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef _LINUX_CPUDL_H -#define _LINUX_CPUDL_H - -#include -#include - -#define IDX_INVALID -1 - -struct cpudl_item { - u64 dl; - int cpu; - int idx; -}; - -struct cpudl { - raw_spinlock_t lock; - int size; - cpumask_var_t free_cpus; - struct cpudl_item *elements; -}; - - -#ifdef CONFIG_SMP -int cpudl_find(struct cpudl *cp, struct task_struct *p, - struct cpumask *later_mask); -void cpudl_set(struct cpudl *cp, int cpu, u64 dl); -void cpudl_clear(struct cpudl *cp, int cpu); -int cpudl_init(struct cpudl *cp); -void cpudl_set_freecpu(struct cpudl *cp, int cpu); -void cpudl_clear_freecpu(struct cpudl *cp, int cpu); -void cpudl_cleanup(struct cpudl *cp); -#endif /* CONFIG_SMP */ - -#endif /* _LINUX_CPUDL_H */ diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c index f0225b133e84..bd7dd17147a1 100644 --- a/kernel/sched/deadline.c +++ b/kernel/sched/deadline.c @@ -265,252 +265,11 @@ void init_dl_rq(struct dl_rq *dl_rq) { dl_rq->rb_root = RB_ROOT; -#ifdef CONFIG_SMP - /* zero means no -deadline tasks */ - dl_rq->earliest_dl.curr = dl_rq->earliest_dl.next = 0; - - dl_rq->dl_nr_migratory = 0; - dl_rq->overloaded = 0; - dl_rq->pushable_dl_tasks_root = RB_ROOT; -#endif - dl_rq->running_bw = 0; dl_rq->this_bw = 0; init_dl_rq_bw_ratio(dl_rq); } -#ifdef CONFIG_SMP - -static inline int dl_overloaded(struct rq *rq) -{ - return atomic_read(&rq->rd->dlo_count); -} - -static inline void dl_set_overload(struct rq *rq) -{ - if (!rq->online) - return; - - cpumask_set_cpu(rq->cpu, rq->rd->dlo_mask); - /* - * Must be visible before the overload count is - * set (as in sched_rt.c). - * - * Matched by the barrier in pull_dl_task(). - */ - smp_wmb(); - atomic_inc(&rq->rd->dlo_count); -} - -static inline void dl_clear_overload(struct rq *rq) -{ - if (!rq->online) - return; - - atomic_dec(&rq->rd->dlo_count); - cpumask_clear_cpu(rq->cpu, rq->rd->dlo_mask); -} - -static void update_dl_migration(struct dl_rq *dl_rq) -{ - if (dl_rq->dl_nr_migratory && dl_rq->dl_nr_running > 1) { - if (!dl_rq->overloaded) { - dl_set_overload(rq_of_dl_rq(dl_rq)); - dl_rq->overloaded = 1; - } - } else if (dl_rq->overloaded) { - dl_clear_overload(rq_of_dl_rq(dl_rq)); - dl_rq->overloaded = 0; - } -} - -static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) -{ - struct task_struct *p = dl_task_of(dl_se); - - if (p->nr_cpus_allowed > 1) - dl_rq->dl_nr_migratory++; - - update_dl_migration(dl_rq); -} - -static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) -{ - struct task_struct *p = dl_task_of(dl_se); - - if (p->nr_cpus_allowed > 1) - dl_rq->dl_nr_migratory--; - - update_dl_migration(dl_rq); -} - -/* - * The list of pushable -deadline task is not a plist, like in - * sched_rt.c, it is an rb-tree with tasks ordered by deadline. - */ -static void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p) -{ - struct dl_rq *dl_rq = &rq->dl; - struct rb_node **link = &dl_rq->pushable_dl_tasks_root.rb_node; - struct rb_node *parent = NULL; - struct task_struct *entry; - int leftmost = 1; - - BUG_ON(!RB_EMPTY_NODE(&p->pushable_dl_tasks)); - - while (*link) { - parent = *link; - entry = rb_entry(parent, struct task_struct, - pushable_dl_tasks); - if (dl_entity_preempt(&p->dl, &entry->dl)) - link = &parent->rb_left; - else { - link = &parent->rb_right; - leftmost = 0; - } - } - - if (leftmost) { - dl_rq->pushable_dl_tasks_leftmost = &p->pushable_dl_tasks; - dl_rq->earliest_dl.next = p->dl.deadline; - } - - rb_link_node(&p->pushable_dl_tasks, parent, link); - rb_insert_color(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root); -} - -static void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p) -{ - struct dl_rq *dl_rq = &rq->dl; - - if (RB_EMPTY_NODE(&p->pushable_dl_tasks)) - return; - - if (dl_rq->pushable_dl_tasks_leftmost == &p->pushable_dl_tasks) { - struct rb_node *next_node; - - next_node = rb_next(&p->pushable_dl_tasks); - dl_rq->pushable_dl_tasks_leftmost = next_node; - if (next_node) { - dl_rq->earliest_dl.next = rb_entry(next_node, - struct task_struct, pushable_dl_tasks)->dl.deadline; - } - } - - rb_erase(&p->pushable_dl_tasks, &dl_rq->pushable_dl_tasks_root); - RB_CLEAR_NODE(&p->pushable_dl_tasks); -} - -static inline int has_pushable_dl_tasks(struct rq *rq) -{ - return !RB_EMPTY_ROOT(&rq->dl.pushable_dl_tasks_root); -} - -static int push_dl_task(struct rq *rq); - -static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev) -{ - return dl_task(prev); -} - -static DEFINE_PER_CPU(struct callback_head, dl_push_head); -static DEFINE_PER_CPU(struct callback_head, dl_pull_head); - -static void push_dl_tasks(struct rq *); -static void pull_dl_task(struct rq *); - -static inline void queue_push_tasks(struct rq *rq) -{ - if (!has_pushable_dl_tasks(rq)) - return; - - queue_balance_callback(rq, &per_cpu(dl_push_head, rq->cpu), push_dl_tasks); -} - -static inline void queue_pull_task(struct rq *rq) -{ - queue_balance_callback(rq, &per_cpu(dl_pull_head, rq->cpu), pull_dl_task); -} - -static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq); - -static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p) -{ - struct rq *later_rq = NULL; - - later_rq = find_lock_later_rq(p, rq); - if (!later_rq) { - int cpu; - - /* - * If we cannot preempt any rq, fall back to pick any - * online cpu. - */ - cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed); - if (cpu >= nr_cpu_ids) { - /* - * Fail to find any suitable cpu. - * The task will never come back! - */ - BUG_ON(dl_bandwidth_enabled()); - - /* - * If admission control is disabled we - * try a little harder to let the task - * run. - */ - cpu = cpumask_any(cpu_active_mask); - } - later_rq = cpu_rq(cpu); - double_lock_balance(rq, later_rq); - } - - set_task_cpu(p, later_rq->cpu); - double_unlock_balance(later_rq, rq); - - return later_rq; -} - -#else - -static inline -void enqueue_pushable_dl_task(struct rq *rq, struct task_struct *p) -{ -} - -static inline -void dequeue_pushable_dl_task(struct rq *rq, struct task_struct *p) -{ -} - -static inline -void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) -{ -} - -static inline -void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) -{ -} - -static inline bool need_pull_dl_task(struct rq *rq, struct task_struct *prev) -{ - return false; -} - -static inline void pull_dl_task(struct rq *rq) -{ -} - -static inline void queue_push_tasks(struct rq *rq) -{ -} - -static inline void queue_pull_task(struct rq *rq) -{ -} -#endif /* CONFIG_SMP */ - static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags); static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags); static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, @@ -905,47 +664,12 @@ static enum hrtimer_restart dl_task_timer(struct hrtimer *timer) goto unlock; } -#ifdef CONFIG_SMP - if (unlikely(!rq->online)) { - /* - * If the runqueue is no longer available, migrate the - * task elsewhere. This necessarily changes rq. - */ - lockdep_unpin_lock(&rq->lock, rf.cookie); - rq = dl_task_offline_migration(rq, p); - rf.cookie = lockdep_pin_lock(&rq->lock); - update_rq_clock(rq); - - /* - * Now that the task has been migrated to the new RQ and we - * have that locked, proceed as normal and enqueue the task - * there. - */ - } -#endif - enqueue_task_dl(rq, p, ENQUEUE_REPLENISH); if (dl_task(rq->curr)) check_preempt_curr_dl(rq, p, 0); else resched_curr(rq); -#ifdef CONFIG_SMP - /* - * Queueing this task back might have overloaded rq, check if we need - * to kick someone away. - */ - if (has_pushable_dl_tasks(rq)) { - /* - * Nothing relies on rq->lock after this, so its safe to drop - * rq->lock. - */ - rq_unpin_lock(rq, &rf); - push_dl_task(rq); - rq_repin_lock(rq, &rf); - } -#endif - unlock: task_rq_unlock(rq, p, &rf); @@ -1176,60 +900,14 @@ void init_dl_inactive_task_timer(struct sched_dl_entity *dl_se) timer->function = inactive_task_timer; } -#ifdef CONFIG_SMP - -static void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) -{ - struct rq *rq = rq_of_dl_rq(dl_rq); - - if (dl_rq->earliest_dl.curr == 0 || - dl_time_before(deadline, dl_rq->earliest_dl.curr)) { - dl_rq->earliest_dl.curr = deadline; - cpudl_set(&rq->rd->cpudl, rq->cpu, deadline); - } -} - -static void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) -{ - struct rq *rq = rq_of_dl_rq(dl_rq); - - /* - * Since we may have removed our earliest (and/or next earliest) - * task we must recompute them. - */ - if (!dl_rq->dl_nr_running) { - dl_rq->earliest_dl.curr = 0; - dl_rq->earliest_dl.next = 0; - cpudl_clear(&rq->rd->cpudl, rq->cpu); - } else { - struct rb_node *leftmost = dl_rq->rb_leftmost; - struct sched_dl_entity *entry; - - entry = rb_entry(leftmost, struct sched_dl_entity, rb_node); - dl_rq->earliest_dl.curr = entry->deadline; - cpudl_set(&rq->rd->cpudl, rq->cpu, entry->deadline); - } -} - -#else - -static inline void inc_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {} -static inline void dec_dl_deadline(struct dl_rq *dl_rq, u64 deadline) {} - -#endif /* CONFIG_SMP */ - static inline void inc_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) { int prio = dl_task_of(dl_se)->prio; - u64 deadline = dl_se->deadline; WARN_ON(!dl_prio(prio)); dl_rq->dl_nr_running++; add_nr_running(rq_of_dl_rq(dl_rq), 1); - - inc_dl_deadline(dl_rq, deadline); - inc_dl_migration(dl_se, dl_rq); } static inline @@ -1241,9 +919,6 @@ void dec_dl_tasks(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) WARN_ON(!dl_rq->dl_nr_running); dl_rq->dl_nr_running--; sub_nr_running(rq_of_dl_rq(dl_rq), 1); - - dec_dl_deadline(dl_rq, dl_se->deadline); - dec_dl_migration(dl_se, dl_rq); } static void __enqueue_dl_entity(struct sched_dl_entity *dl_se) @@ -1384,15 +1059,11 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags) } enqueue_dl_entity(&p->dl, pi_se, flags); - - if (!task_current(rq, p) && p->nr_cpus_allowed > 1) - enqueue_pushable_dl_task(rq, p); } static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags) { dequeue_dl_entity(&p->dl); - dequeue_pushable_dl_task(rq, p); } static void dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags) @@ -1455,105 +1126,6 @@ static void yield_task_dl(struct rq *rq) rq_clock_skip_update(rq, true); } -#ifdef CONFIG_SMP - -static int find_later_rq(struct task_struct *task); - -static int -select_task_rq_dl(struct task_struct *p, int cpu, int sd_flag, int flags) -{ - struct task_struct *curr; - struct rq *rq; - - if (sd_flag != SD_BALANCE_WAKE) - goto out; - - rq = cpu_rq(cpu); - - rcu_read_lock(); - curr = READ_ONCE(rq->curr); /* unlocked access */ - - /* - * If we are dealing with a -deadline task, we must - * decide where to wake it up. - * If it has a later deadline and the current task - * on this rq can't move (provided the waking task - * can!) we prefer to send it somewhere else. On the - * other hand, if it has a shorter deadline, we - * try to make it stay here, it might be important. - */ - if (unlikely(dl_task(curr)) && - (curr->nr_cpus_allowed < 2 || - !dl_entity_preempt(&p->dl, &curr->dl)) && - (p->nr_cpus_allowed > 1)) { - int target = find_later_rq(p); - - if (target != -1 && - (dl_time_before(p->dl.deadline, - cpu_rq(target)->dl.earliest_dl.curr) || - (cpu_rq(target)->dl.dl_nr_running == 0))) - cpu = target; - } - rcu_read_unlock(); - -out: - return cpu; -} - -static void migrate_task_rq_dl(struct task_struct *p) -{ - struct rq *rq; - - if (p->state != TASK_WAKING) - return; - - rq = task_rq(p); - /* - * Since p->state == TASK_WAKING, set_task_cpu() has been called - * from try_to_wake_up(). Hence, p->pi_lock is locked, but - * rq->lock is not... So, lock it - */ - raw_spin_lock(&rq->lock); - if (p->dl.dl_non_contending) { - sub_running_bw(p->dl.dl_bw, &rq->dl); - p->dl.dl_non_contending = 0; - /* - * If the timer handler is currently running and the - * timer cannot be cancelled, inactive_task_timer() - * will see that dl_not_contending is not set, and - * will not touch the rq's active utilization, - * so we are still safe. - */ - if (hrtimer_try_to_cancel(&p->dl.inactive_timer) == 1) - put_task_struct(p); - } - sub_rq_bw(p->dl.dl_bw, &rq->dl); - raw_spin_unlock(&rq->lock); -} - -static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p) -{ - /* - * Current can't be migrated, useless to reschedule, - * let's hope p can move out. - */ - if (rq->curr->nr_cpus_allowed == 1 || - cpudl_find(&rq->rd->cpudl, rq->curr, NULL) == -1) - return; - - /* - * p is migratable, so let's not schedule it and - * see if it is pushed or pulled somewhere else. - */ - if (p->nr_cpus_allowed != 1 && - cpudl_find(&rq->rd->cpudl, p, NULL) != -1) - return; - - resched_curr(rq); -} - -#endif /* CONFIG_SMP */ - /* * Only called when both the current and waking task are -deadline * tasks. @@ -1565,16 +1137,6 @@ static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, resched_curr(rq); return; } - -#ifdef CONFIG_SMP - /* - * In the unlikely case current and p have the same deadline - * let us try to decide what's the best thing to do... - */ - if ((p->dl.deadline == rq->curr->dl.deadline) && - !test_tsk_need_resched(rq->curr)) - check_preempt_equal_dl(rq, p); -#endif /* CONFIG_SMP */ } #ifdef CONFIG_SCHED_HRTICK @@ -1608,25 +1170,6 @@ pick_next_task_dl(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) dl_rq = &rq->dl; - if (need_pull_dl_task(rq, prev)) { - /* - * This is OK, because current is on_cpu, which avoids it being - * picked for load-balance and preemption/IRQs are still - * disabled avoiding further scheduler activity on it and we're - * being very careful to re-start the picking loop. - */ - rq_unpin_lock(rq, rf); - pull_dl_task(rq); - rq_repin_lock(rq, rf); - /* - * pull_dl_task() can drop (and re-acquire) rq->lock; this - * means a stop task can slip in, in which case we need to - * re-start task selection. - */ - if (rq->stop && task_on_rq_queued(rq->stop)) - return RETRY_TASK; - } - /* * When prev is DL, we may throttle it in put_prev_task(). * So, we update time before we check for dl_nr_running. @@ -1645,23 +1188,15 @@ pick_next_task_dl(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) p = dl_task_of(dl_se); p->se.exec_start = rq_clock_task(rq); - /* Running task will never be pushed. */ - dequeue_pushable_dl_task(rq, p); - if (hrtick_enabled(rq)) start_hrtick_dl(rq, p); - queue_push_tasks(rq); - return p; } static void put_prev_task_dl(struct rq *rq, struct task_struct *p) { update_curr_dl(rq); - - if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1) - enqueue_pushable_dl_task(rq, p); } static void task_tick_dl(struct rq *rq, struct task_struct *p, int queued) @@ -1691,410 +1226,9 @@ static void set_curr_task_dl(struct rq *rq) struct task_struct *p = rq->curr; p->se.exec_start = rq_clock_task(rq); - - /* You can't push away the running task */ - dequeue_pushable_dl_task(rq, p); } #ifdef CONFIG_SMP - -/* Only try algorithms three times */ -#define DL_MAX_TRIES 3 - -static int pick_dl_task(struct rq *rq, struct task_struct *p, int cpu) -{ - if (!task_running(rq, p) && - cpumask_test_cpu(cpu, &p->cpus_allowed)) - return 1; - return 0; -} - -/* - * Return the earliest pushable rq's task, which is suitable to be executed - * on the CPU, NULL otherwise: - */ -static struct task_struct *pick_earliest_pushable_dl_task(struct rq *rq, int cpu) -{ - struct rb_node *next_node = rq->dl.pushable_dl_tasks_leftmost; - struct task_struct *p = NULL; - - if (!has_pushable_dl_tasks(rq)) - return NULL; - -next_node: - if (next_node) { - p = rb_entry(next_node, struct task_struct, pushable_dl_tasks); - - if (pick_dl_task(rq, p, cpu)) - return p; - - next_node = rb_next(next_node); - goto next_node; - } - - return NULL; -} - -static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask_dl); - -static int find_later_rq(struct task_struct *task) -{ - struct sched_domain *sd; - struct cpumask *later_mask = this_cpu_cpumask_var_ptr(local_cpu_mask_dl); - int this_cpu = smp_processor_id(); - int best_cpu, cpu = task_cpu(task); - - /* Make sure the mask is initialized first */ - if (unlikely(!later_mask)) - return -1; - - if (task->nr_cpus_allowed == 1) - return -1; - - /* - * We have to consider system topology and task affinity - * first, then we can look for a suitable cpu. - */ - best_cpu = cpudl_find(&task_rq(task)->rd->cpudl, - task, later_mask); - if (best_cpu == -1) - return -1; - - /* - * If we are here, some target has been found, - * the most suitable of which is cached in best_cpu. - * This is, among the runqueues where the current tasks - * have later deadlines than the task's one, the rq - * with the latest possible one. - * - * Now we check how well this matches with task's - * affinity and system topology. - * - * The last cpu where the task run is our first - * guess, since it is most likely cache-hot there. - */ - if (cpumask_test_cpu(cpu, later_mask)) - return cpu; - /* - * Check if this_cpu is to be skipped (i.e., it is - * not in the mask) or not. - */ - if (!cpumask_test_cpu(this_cpu, later_mask)) - this_cpu = -1; - - rcu_read_lock(); - for_each_domain(cpu, sd) { - if (sd->flags & SD_WAKE_AFFINE) { - - /* - * If possible, preempting this_cpu is - * cheaper than migrating. - */ - if (this_cpu != -1 && - cpumask_test_cpu(this_cpu, sched_domain_span(sd))) { - rcu_read_unlock(); - return this_cpu; - } - - /* - * Last chance: if best_cpu is valid and is - * in the mask, that becomes our choice. - */ - if (best_cpu < nr_cpu_ids && - cpumask_test_cpu(best_cpu, sched_domain_span(sd))) { - rcu_read_unlock(); - return best_cpu; - } - } - } - rcu_read_unlock(); - - /* - * At this point, all our guesses failed, we just return - * 'something', and let the caller sort the things out. - */ - if (this_cpu != -1) - return this_cpu; - - cpu = cpumask_any(later_mask); - if (cpu < nr_cpu_ids) - return cpu; - - return -1; -} - -/* Locks the rq it finds */ -static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq) -{ - struct rq *later_rq = NULL; - int tries; - int cpu; - - for (tries = 0; tries < DL_MAX_TRIES; tries++) { - cpu = find_later_rq(task); - - if ((cpu == -1) || (cpu == rq->cpu)) - break; - - later_rq = cpu_rq(cpu); - - if (later_rq->dl.dl_nr_running && - !dl_time_before(task->dl.deadline, - later_rq->dl.earliest_dl.curr)) { - /* - * Target rq has tasks of equal or earlier deadline, - * retrying does not release any lock and is unlikely - * to yield a different result. - */ - later_rq = NULL; - break; - } - - /* Retry if something changed. */ - if (double_lock_balance(rq, later_rq)) { - if (unlikely(task_rq(task) != rq || - !cpumask_test_cpu(later_rq->cpu, &task->cpus_allowed) || - task_running(rq, task) || - !dl_task(task) || - !task_on_rq_queued(task))) { - double_unlock_balance(rq, later_rq); - later_rq = NULL; - break; - } - } - - /* - * If the rq we found has no -deadline task, or - * its earliest one has a later deadline than our - * task, the rq is a good one. - */ - if (!later_rq->dl.dl_nr_running || - dl_time_before(task->dl.deadline, - later_rq->dl.earliest_dl.curr)) - break; - - /* Otherwise we try again. */ - double_unlock_balance(rq, later_rq); - later_rq = NULL; - } - - return later_rq; -} - -static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) -{ - struct task_struct *p; - - if (!has_pushable_dl_tasks(rq)) - return NULL; - - p = rb_entry(rq->dl.pushable_dl_tasks_leftmost, - struct task_struct, pushable_dl_tasks); - - BUG_ON(rq->cpu != task_cpu(p)); - BUG_ON(task_current(rq, p)); - BUG_ON(p->nr_cpus_allowed <= 1); - - BUG_ON(!task_on_rq_queued(p)); - BUG_ON(!dl_task(p)); - - return p; -} - -/* - * See if the non running -deadline tasks on this rq - * can be sent to some other CPU where they can preempt - * and start executing. - */ -static int push_dl_task(struct rq *rq) -{ - struct task_struct *next_task; - struct rq *later_rq; - int ret = 0; - - if (!rq->dl.overloaded) - return 0; - - next_task = pick_next_pushable_dl_task(rq); - if (!next_task) - return 0; - -retry: - if (unlikely(next_task == rq->curr)) { - WARN_ON(1); - return 0; - } - - /* - * If next_task preempts rq->curr, and rq->curr - * can move away, it makes sense to just reschedule - * without going further in pushing next_task. - */ - if (dl_task(rq->curr) && - dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) && - rq->curr->nr_cpus_allowed > 1) { - resched_curr(rq); - return 0; - } - - /* We might release rq lock */ - get_task_struct(next_task); - - /* Will lock the rq it'll find */ - later_rq = find_lock_later_rq(next_task, rq); - if (!later_rq) { - struct task_struct *task; - - /* - * We must check all this again, since - * find_lock_later_rq releases rq->lock and it is - * then possible that next_task has migrated. - */ - task = pick_next_pushable_dl_task(rq); - if (task == next_task) { - /* - * The task is still there. We don't try - * again, some other cpu will pull it when ready. - */ - goto out; - } - - if (!task) - /* No more tasks */ - goto out; - - put_task_struct(next_task); - next_task = task; - goto retry; - } - - deactivate_task(rq, next_task, 0); - sub_running_bw(next_task->dl.dl_bw, &rq->dl); - sub_rq_bw(next_task->dl.dl_bw, &rq->dl); - set_task_cpu(next_task, later_rq->cpu); - add_rq_bw(next_task->dl.dl_bw, &later_rq->dl); - add_running_bw(next_task->dl.dl_bw, &later_rq->dl); - activate_task(later_rq, next_task, 0); - ret = 1; - - resched_curr(later_rq); - - double_unlock_balance(rq, later_rq); - -out: - put_task_struct(next_task); - - return ret; -} - -static void push_dl_tasks(struct rq *rq) -{ - /* push_dl_task() will return true if it moved a -deadline task */ - while (push_dl_task(rq)) - ; -} - -static void pull_dl_task(struct rq *this_rq) -{ - int this_cpu = this_rq->cpu, cpu; - struct task_struct *p; - bool resched = false; - struct rq *src_rq; - u64 dmin = LONG_MAX; - - if (likely(!dl_overloaded(this_rq))) - return; - - /* - * Match the barrier from dl_set_overloaded; this guarantees that if we - * see overloaded we must also see the dlo_mask bit. - */ - smp_rmb(); - - for_each_cpu(cpu, this_rq->rd->dlo_mask) { - if (this_cpu == cpu) - continue; - - src_rq = cpu_rq(cpu); - - /* - * It looks racy, abd it is! However, as in sched_rt.c, - * we are fine with this. - */ - if (this_rq->dl.dl_nr_running && - dl_time_before(this_rq->dl.earliest_dl.curr, - src_rq->dl.earliest_dl.next)) - continue; - - /* Might drop this_rq->lock */ - double_lock_balance(this_rq, src_rq); - - /* - * If there are no more pullable tasks on the - * rq, we're done with it. - */ - if (src_rq->dl.dl_nr_running <= 1) - goto skip; - - p = pick_earliest_pushable_dl_task(src_rq, this_cpu); - - /* - * We found a task to be pulled if: - * - it preempts our current (if there's one), - * - it will preempt the last one we pulled (if any). - */ - if (p && dl_time_before(p->dl.deadline, dmin) && - (!this_rq->dl.dl_nr_running || - dl_time_before(p->dl.deadline, - this_rq->dl.earliest_dl.curr))) { - WARN_ON(p == src_rq->curr); - WARN_ON(!task_on_rq_queued(p)); - - /* - * Then we pull iff p has actually an earlier - * deadline than the current task of its runqueue. - */ - if (dl_time_before(p->dl.deadline, - src_rq->curr->dl.deadline)) - goto skip; - - resched = true; - - deactivate_task(src_rq, p, 0); - sub_running_bw(p->dl.dl_bw, &src_rq->dl); - sub_rq_bw(p->dl.dl_bw, &src_rq->dl); - set_task_cpu(p, this_cpu); - add_rq_bw(p->dl.dl_bw, &this_rq->dl); - add_running_bw(p->dl.dl_bw, &this_rq->dl); - activate_task(this_rq, p, 0); - dmin = p->dl.deadline; - - /* Is there any other task even earlier? */ - } -skip: - double_unlock_balance(this_rq, src_rq); - } - - if (resched) - resched_curr(this_rq); -} - -/* - * Since the task is not running and a reschedule is not going to happen - * anytime soon on its runqueue, we try pushing it away now. - */ -static void task_woken_dl(struct rq *rq, struct task_struct *p) -{ - if (!task_running(rq, p) && - !test_tsk_need_resched(rq->curr) && - p->nr_cpus_allowed > 1 && - dl_task(rq->curr) && - (rq->curr->nr_cpus_allowed < 2 || - !dl_entity_preempt(&p->dl, &rq->curr->dl))) { - push_dl_tasks(rq); - } -} - static void set_cpus_allowed_dl(struct task_struct *p, const struct cpumask *new_mask) { @@ -2102,38 +1236,7 @@ static void set_cpus_allowed_dl(struct task_struct *p, set_cpus_allowed_common(p, new_mask); } - -/* Assumes rq->lock is held */ -static void rq_online_dl(struct rq *rq) -{ - if (rq->dl.overloaded) - dl_set_overload(rq); - - cpudl_set_freecpu(&rq->rd->cpudl, rq->cpu); - if (rq->dl.dl_nr_running > 0) - cpudl_set(&rq->rd->cpudl, rq->cpu, rq->dl.earliest_dl.curr); -} - -/* Assumes rq->lock is held */ -static void rq_offline_dl(struct rq *rq) -{ - if (rq->dl.overloaded) - dl_clear_overload(rq); - - cpudl_clear(&rq->rd->cpudl, rq->cpu); - cpudl_clear_freecpu(&rq->rd->cpudl, rq->cpu); -} - -void __init init_sched_dl_class(void) -{ - unsigned int i; - - for_each_possible_cpu(i) - zalloc_cpumask_var_node(&per_cpu(local_cpu_mask_dl, i), - GFP_KERNEL, cpu_to_node(i)); -} - -#endif /* CONFIG_SMP */ +#endif static void switched_from_dl(struct rq *rq, struct task_struct *p) { @@ -2158,16 +1261,6 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p) */ if (p->dl.dl_non_contending) p->dl.dl_non_contending = 0; - - /* - * Since this might be the only -deadline task on the rq, - * this is the right place to try to pull some other one - * from an overloaded cpu, if any. - */ - if (!task_on_rq_queued(p) || rq->dl.dl_nr_running) - return; - - queue_pull_task(rq); } /* @@ -2194,10 +1287,6 @@ static void switched_to_dl(struct rq *rq, struct task_struct *p) setup_new_dl_entity(&p->dl); if (rq->curr != p) { -#ifdef CONFIG_SMP - if (p->nr_cpus_allowed > 1 && rq->dl.overloaded) - queue_push_tasks(rq); -#endif if (dl_task(rq->curr)) check_preempt_curr_dl(rq, p, 0); else @@ -2213,31 +1302,6 @@ static void prio_changed_dl(struct rq *rq, struct task_struct *p, int oldprio) { if (task_on_rq_queued(p) || rq->curr == p) { -#ifdef CONFIG_SMP - /* - * This might be too much, but unfortunately - * we don't have the old deadline value, and - * we can't argue if the task is increasing - * or lowering its prio, so... - */ - if (!rq->dl.overloaded) - queue_pull_task(rq); - - /* - * If we now have a earlier deadline task than p, - * then reschedule, provided p is still on this - * runqueue. - */ - if (dl_time_before(rq->dl.earliest_dl.curr, p->dl.deadline)) - resched_curr(rq); -#else - /* - * Again, we don't know if p has a earlier - * or later deadline, so let's blindly set a - * (maybe not needed) rescheduling point. - */ - resched_curr(rq); -#endif /* CONFIG_SMP */ } } @@ -2253,12 +1317,7 @@ const struct sched_class dl_sched_class = { .put_prev_task = put_prev_task_dl, #ifdef CONFIG_SMP - .select_task_rq = select_task_rq_dl, - .migrate_task_rq = migrate_task_rq_dl, .set_cpus_allowed = set_cpus_allowed_dl, - .rq_online = rq_online_dl, - .rq_offline = rq_offline_dl, - .task_woken = task_woken_dl, #endif .set_curr_task = set_curr_task_dl, diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c index 99e76d5439c4..9e87167ccfe3 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -581,7 +581,6 @@ void print_dl_rq(struct seq_file *m, int cpu, struct dl_rq *dl_rq) PU(dl_nr_running); #ifdef CONFIG_SMP - PU(dl_nr_migratory); //dl_bw = &cpu_rq(cpu)->rd->dl_bw; #else //dl_bw = &dl_rq->dl_bw; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 3c27e73e468f..f77531d51e9a 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -35,7 +35,6 @@ #endif #include "cpupri.h" -#include "cpudeadline.h" #include "cpuacct.h" #ifdef CONFIG_SCHED_DEBUG @@ -523,29 +522,6 @@ struct dl_rq { unsigned long dl_nr_running; -#ifdef CONFIG_SMP - /* - * Deadline values of the currently executing and the - * earliest ready task on this rq. Caching these facilitates - * the decision wether or not a ready but not running task - * should migrate somewhere else. - */ - struct { - u64 curr; - u64 next; - } earliest_dl; - - unsigned long dl_nr_migratory; - int overloaded; - - /* - * Tasks on this rq that can be pushed away. They are kept in - * an rb-tree, ordered by tasks' deadlines, with caching - * of the leftmost (earliest deadline) element. - */ - struct rb_root pushable_dl_tasks_root; - struct rb_node *pushable_dl_tasks_leftmost; -#endif /* * "Active utilization" for this runqueue: increased when a * task wakes up (becomes TASK_RUNNING) and decreased when a @@ -598,14 +574,6 @@ struct root_domain { bool overload; /* - * The bit corresponding to a CPU gets set here if such CPU has more - * than one runnable -deadline task (as it is below for RT tasks). - */ - cpumask_var_t dlo_mask; - atomic_t dlo_count; - struct cpudl cpudl; - - /* * The "RT overload" flag: it gets set if a CPU has more than * one runnable RT task. */ diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index db8afd7930f1..b6b966683e8f 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -214,8 +214,6 @@ static void free_rootdomain(struct rcu_head *rcu) struct root_domain *rd = container_of(rcu, struct root_domain, rcu); cpupri_cleanup(&rd->cpupri); - cpudl_cleanup(&rd->cpudl); - free_cpumask_var(rd->dlo_mask); free_cpumask_var(rd->rto_mask); free_cpumask_var(rd->online); free_cpumask_var(rd->span); @@ -267,24 +265,15 @@ static int init_rootdomain(struct root_domain *rd) goto out; if (!zalloc_cpumask_var(&rd->online, GFP_KERNEL)) goto free_span; - if (!zalloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL)) - goto free_online; if (!zalloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) - goto free_dlo_mask; - - if (cpudl_init(&rd->cpudl) != 0) - goto free_rto_mask; + goto free_online; if (cpupri_init(&rd->cpupri) != 0) - goto free_cpudl; + goto free_rto_mask; return 0; -free_cpudl: - cpudl_cleanup(&rd->cpudl); free_rto_mask: free_cpumask_var(rd->rto_mask); -free_dlo_mask: - free_cpumask_var(rd->dlo_mask); free_online: free_cpumask_var(rd->online); free_span: -- 2.11.0