s390/idle: Slightly optimize idle time accounting

Slightly optimize account_idle_time_irq() and update_timer_idle():

- Use fast single instruction __atomic64() primitives to update per
  cpu idle_time and idle_count, instead of READ_ONCE() / WRITE_ONCE()
  pairs

- stcctm() is an inline assembly with a full memory barrier. This
  leads to a not necessary extra dereference of smp_cpu_mtid in
  update_timer_idle(). Avoid this and read smp_cpu_mtid into a
  variable

- Use __this_cpu_add() instead of this_cpu_add() to avoid disabling /
  enabling of preemption several times in a loop in update_timer_idle().

Reviewed-by: Sven Schnelle <svens@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

diff --git a/arch/s390/kernel/idle.c b/arch/s390/kernel/idle.c
index 4e09f126d4fc5f63a874b70e3ec1e818c59e8311..627d82dd900ea12c2b152567031c49a3e048795f 100644 (file)
--- a/arch/s390/kernel/idle.c
+++ b/arch/s390/kernel/idle.c
@@ -26,12 +26,13 @@ void update_timer_idle(void)
        struct s390_idle_data *idle = this_cpu_ptr(&s390_idle);
        struct lowcore *lc = get_lowcore();
        u64 cycles_new[8];
-       int i;
+       int i, mtid;
 
-       if (smp_cpu_mtid) {
-               stcctm(MT_DIAG, smp_cpu_mtid, cycles_new);
-               for (i = 0; i < smp_cpu_mtid; i++)
-                       this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]);
+       mtid = smp_cpu_mtid;
+       if (mtid) {
+               stcctm(MT_DIAG, mtid, cycles_new);
+               for (i = 0; i < mtid; i++)
+                       __this_cpu_add(mt_cycles[i], cycles_new[i] - idle->mt_cycles_enter[i]);
        }
 
        /*
@@ -58,8 +59,8 @@ void account_idle_time_irq(void)
        idle_time = get_lowcore()->int_clock - idle->clock_idle_enter;
 
        /* Account time spent with enabled wait psw loaded as idle time. */
-       WRITE_ONCE(idle->idle_time, READ_ONCE(idle->idle_time) + idle_time);
-       WRITE_ONCE(idle->idle_count, READ_ONCE(idle->idle_count) + 1);
+       __atomic64_add(idle_time, &idle->idle_time);
+       __atomic64_add_const(1, &idle->idle_count);
        account_idle_time(cputime_to_nsecs(idle_time));
 }