From: Nick Piggin <nickpiggin@yahoo.com.au>
Add some likely/unliklies, a for_each_cpu => for_each_cpu_online, and close
the sched_exit race.
Signed-off-by: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/kernel/sched.c | 20 +++++++++-----------
1 files changed, 9 insertions(+), 11 deletions(-)
diff -puN kernel/sched.c~sched-misc kernel/sched.c
--- 25/kernel/sched.c~sched-misc 2004-07-13 13:17:29.783906688 -0700
+++ 25-akpm/kernel/sched.c 2004-07-13 13:17:29.789905776 -0700
@@ -942,7 +942,7 @@ void fastcall sched_fork(task_t *p, unsi
p->first_time_slice = 1;
current->time_slice >>= 1;
p->timestamp = sched_clock();
- if (!current->time_slice) {
+ if (unlikely(!current->time_slice)) {
/*
* This case is rare, it happens when the parent has only
* a single jiffy left from its timeslice. Taking the
@@ -1049,18 +1049,16 @@ void fastcall sched_exit(task_t * p)
unsigned long flags;
runqueue_t *rq;
- local_irq_save(flags);
- if (p->first_time_slice) {
- p->parent->time_slice += p->time_slice;
- if (unlikely(p->parent->time_slice > MAX_TIMESLICE))
- p->parent->time_slice = MAX_TIMESLICE;
- }
- local_irq_restore(flags);
/*
* If the child was a (relative-) CPU hog then decrease
* the sleep_avg of the parent as well.
*/
rq = task_rq_lock(p->parent, &flags);
+ if (p->first_time_slice) {
+ p->parent->time_slice += p->time_slice;
+ if (unlikely(p->parent->time_slice > MAX_TIMESLICE))
+ p->parent->time_slice = MAX_TIMESLICE;
+ }
if (p->sleep_avg < p->parent->sleep_avg)
p->parent->sleep_avg = p->parent->sleep_avg /
(EXIT_WEIGHT + 1) * EXIT_WEIGHT + p->sleep_avg /
@@ -1160,7 +1158,7 @@ unsigned long nr_running(void)
{
unsigned long i, sum = 0;
- for_each_cpu(i)
+ for_each_online_cpu(i)
sum += cpu_rq(i)->nr_running;
return sum;
@@ -2273,7 +2271,7 @@ need_resched:
goto switch_tasks;
}
- if (!rt_task(next) && next->activated > 0) {
+ if (rt_task(next) && next->activated > 0) {
unsigned long long delta = now - next->timestamp;
if (next->activated == 1)
@@ -2314,7 +2312,7 @@ switch_tasks:
reacquire_kernel_lock(current);
preempt_enable_no_resched();
- if (test_thread_flag(TIF_NEED_RESCHED))
+ if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
goto need_resched;
}
_