Here is a new trick.
Labeling previous implementation (assume system applied [1/2] only) as V.1,
it does:
- At exit of kernel (ia64_leave_kernel), calculate cycles from last
check point using last stamp (ac_stamp), then accumulate the cycle
as "system's cycles" (ac_stime) and updates the stamp.
- At entrance of kernel (break_fault etc.), calculate cycles from last
check point using last stamp, then accumulate the cycle as "user's
cycles" (ac_utime) and updates the stamp.
It takes times from both of kernel entrance path and exit path, so it
results in considerable increase of system call overhead, unfortunately.
This 2 of 2 is magical patch to reduce the overhead.
Labeling new implementation (assume system applied [1/2] and [2/2]) as V.2,
it does:
- At exit of kernel (ia64_leave_kernel), do nothing but only save the
"leave time" as ac_leave separated from usual time stamp (ac_stamp).
- At entrance of kernel (break_fault etc.):
1. calculate cycles from last check point (in kernel) to "last leave"
using ac_stamp and ac_leave, then accumulate the cycle to ac_stime.
2. calculate cycles from "last leave" to now, then accumulate the
cycle to ac_utime.
3. updates the stamp (ac_stamp).
It can be said that this patch combines most part of separated jobs and
moves it to entrance side. The change is simple, however:
- Exit path becomes quite simple. Only needed is posting value of ITC
to memory. There were few registers and slots in bundles available
for extra work, but fortunately I made it without increasing the
number of bundles ;-)
- Entry path becomes slightly complicated. But we can load/store data
at once, and do not need to do it at both of exit/entrance anymore.
Following result of benchmark shows the performance impact of my patches.
(V.1 = 2.6.24-rc5 + [1/2], V.2 = rc5 + [1/2] + [2/2], orig. = rc5)
===========================================================================
INDEX VALUES RATIO(%)
TEST (Unixbench-v4.1.0) V.1 V.2 orig. V.1 V.2 orig.
=============================== ====== ====== ====== ===== ===== =====
Dhrystone 2 using register var. 304.3 304.3 304.4 100.0 100.0 100.0
Double-Precision Whetstone 171.3 171.3 171.1 100.1 100.1 100.0
Execl Throughput 471.3 466.2 467.3 100.9 99.8 100.0
File Copy 1024 buf 2000 maxblks 496.6 511.1 507.1 97.9 100.8 100.0
File Copy 256 buf 500 maxblks 352.4 355.3 366.2 96.2 97.0 100.0
File Copy 4096 buf 8000 maxblks 765.8 768.6 778.1 98.4 98.8 100.0
Pipe Throughput 422.0 427.0 416.1 101.4 102.6 100.0
Process Creation 945.1 949.3 948.0 99.7 100.1 100.0
Shell Scripts (8 concurrent) 1646.7 1646.2 1654.5 99.5 99.5 100.0
System Call Overhead 695.4 732.1 820.0 84.8 89.3 100.0
=============================== ====== ====== ====== ===== ===== =====
FINAL SCORE 522.0 527.1 533.9 97.8 98.7 100.0
===========================================================================
(@ Madison 1.5GHz x 4)
Most affected is undoubtedly system call, which is originally well optimized.
But from macro-viewpoint, unless you are a full-time-system-call-aholic, I
believe it's worthwhile to make a concession.
The faster hardware goes (or even the more active software becomes), the
more accuracy of traditional tick-sampling based CPU time accounting drops.
When would be the decision point?
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@xxxxxxxxxxxxxx>
---
arch/ia64/kernel/asm-offsets.c | 1
arch/ia64/kernel/entry.S | 87 ++++++++++++++++++++++++++++++-----------
arch/ia64/kernel/fsys.S | 20 ++++++---
arch/ia64/kernel/ivt.S | 42 ++++++++++++-------
include/asm-ia64/thread_info.h | 1
5 files changed, 107 insertions(+), 44 deletions(-)
Index: linux-2.6.24-rc5/arch/ia64/kernel/asm-offsets.c
===================================================================
--- linux-2.6.24-rc5.orig/arch/ia64/kernel/asm-offsets.c
+++ linux-2.6.24-rc5/arch/ia64/kernel/asm-offsets.c
@@ -41,6 +41,7 @@
DEFINE(TI_PRE_COUNT, offsetof(struct thread_info, preempt_count));
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
DEFINE(TI_AC_STAMP, offsetof(struct thread_info, ac_stamp));
+ DEFINE(TI_AC_LEAVE, offsetof(struct thread_info, ac_leave));
DEFINE(TI_AC_STIME, offsetof(struct thread_info, ac_stime));
DEFINE(TI_AC_UTIME, offsetof(struct thread_info, ac_utime));
#endif
Index: linux-2.6.24-rc5/arch/ia64/kernel/entry.S
===================================================================
--- linux-2.6.24-rc5.orig/arch/ia64/kernel/entry.S
+++ linux-2.6.24-rc5/arch/ia64/kernel/entry.S
@@ -710,6 +710,16 @@
(pUStk) cmp.eq.unc p6,p0=r0,r0 // p6 <- pUStk
#endif
.work_processed_syscall:
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+ adds r2=PT(LOADRS)+16,r12
+(pUStk) mov.m r22=ar.itc // fetch time at leave
+ adds r18=TI_FLAGS+IA64_TASK_SIZE,r13
+ ;;
+(p6) ld4 r31=[r18] // load current_thread_info()->flags
+ ld8 r19=[r2],PT(B6)-PT(LOADRS) // load ar.rsc value for "loadrs"
+ adds r3=PT(AR_BSPSTORE)+16,r12 // deferred
+ ;;
+#else
adds r2=PT(LOADRS)+16,r12
adds r3=PT(AR_BSPSTORE)+16,r12
adds r18=TI_FLAGS+IA64_TASK_SIZE,r13
@@ -718,6 +728,7 @@
ld8 r19=[r2],PT(B6)-PT(LOADRS) // load ar.rsc value for "loadrs"
nop.i 0
;;
+#endif
mov r16=ar.bsp // M2 get existing backing store pointer
ld8 r18=[r2],PT(R9)-PT(B6) // load b6
(p6) and r15=TIF_WORK_MASK,r31 // any work other than TIF_SYSCALL_TRACE?
@@ -737,12 +748,21 @@
ld8 r29=[r2],16 // M0|1 load cr.ipsr
ld8 r28=[r3],16 // M0|1 load cr.iip
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+(pUStk) add r14=TI_AC_LEAVE+IA64_TASK_SIZE,r13
+ ;;
+ ld8 r30=[r2],16 // M0|1 load cr.ifs
+ ld8 r25=[r3],16 // M0|1 load ar.unat
+(pUStk) add r15=IA64_TASK_THREAD_ON_USTACK_OFFSET,r13
+ ;;
+#else
mov r22=r0 // A clear r22
;;
ld8 r30=[r2],16 // M0|1 load cr.ifs
ld8 r25=[r3],16 // M0|1 load ar.unat
(pUStk) add r14=IA64_TASK_THREAD_ON_USTACK_OFFSET,r13
;;
+#endif
ld8 r26=[r2],PT(B0)-PT(AR_PFS) // M0|1 load ar.pfs
(pKStk) mov r22=psr // M2 read PSR now that interrupts are disabled
nop 0
@@ -759,7 +779,11 @@
ld8.fill r1=[r3],16 // M0|1 load r1
(pUStk) mov r17=1 // A
;;
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+(pUStk) st1 [r15]=r17 // M2|3
+#else
(pUStk) st1 [r14]=r17 // M2|3
+#endif
ld8.fill r13=[r3],16 // M0|1
mov f8=f0 // F clear f8
;;
@@ -775,12 +799,22 @@
shr.u r18=r19,16 // I0|1 get byte size of existing "dirty" partition
cover // B add current frame into dirty partition & set cr.ifs
;;
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+ mov r19=ar.bsp // M2 get new backing store pointer
+ st8 [r14]=r22 // M save time at leave
+ mov f10=f0 // F clear f10
+
+ mov r22=r0 // A clear r22
+ movl r14=__kernel_syscall_via_epc // X
+ ;;
+#else
mov r19=ar.bsp // M2 get new backing store pointer
mov f10=f0 // F clear f10
nop.m 0
movl r14=__kernel_syscall_via_epc // X
;;
+#endif
mov.m ar.csd=r0 // M2 clear ar.csd
mov.m ar.ccv=r0 // M2 clear ar.ccv
mov b7=r14 // I0 clear b7 (hint with __kernel_syscall_via_epc)
@@ -913,10 +947,18 @@
adds r16=PT(CR_IPSR)+16,r12
adds r17=PT(CR_IIP)+16,r12
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+ .pred.rel.mutex pUStk,pKStk
+(pKStk) mov r22=psr // M2 read PSR now that interrupts are disabled
+(pUStk) mov.m r22=ar.itc // M fetch time at leave
+ nop.i 0
+ ;;
+#else
(pKStk) mov r22=psr // M2 read PSR now that interrupts are disabled
nop.i 0
nop.i 0
;;
+#endif
ld8 r29=[r16],16 // load cr.ipsr
ld8 r28=[r17],16 // load cr.iip
;;
@@ -938,15 +980,37 @@
;;
ld8.fill r12=[r16],16
ld8.fill r13=[r17],16
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+(pUStk) adds r3=TI_AC_LEAVE+IA64_TASK_SIZE,r18
+#else
(pUStk) adds r18=IA64_TASK_THREAD_ON_USTACK_OFFSET,r18
+#endif
;;
ld8 r20=[r16],16 // ar.fpsr
ld8.fill r15=[r17],16
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+(pUStk) adds r18=IA64_TASK_THREAD_ON_USTACK_OFFSET,r18 // deferred
+#endif
;;
ld8.fill r14=[r16],16
ld8.fill r2=[r17]
(pUStk) mov r17=1
;;
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+ // mmi_ : ld8 st1 shr;; mmi_ : st8 st1 shr;;
+ // mib : mov add br -> mib : ld8 add br
+ // bbb_ : br nop cover;; mbb_ : mov br cover;;
+ //
+ // no one require bsp in r16 if (pKStk) branch is selected.
+(pUStk) st8 [r3]=r22 // save time at leave
+(pUStk) st1 [r18]=r17 // restore current->thread.on_ustack
+ shr.u r18=r19,16 // get byte size of existing "dirty" partition
+ ;;
+ ld8.fill r3=[r16] // deferred
+ LOAD_PHYS_STACK_REG_SIZE(r17)
+(pKStk) br.cond.dpnt skip_rbs_switch
+ mov r16=ar.bsp // get existing backing store pointer
+#else
ld8.fill r3=[r16]
(pUStk) st1 [r18]=r17 // restore current->thread.on_ustack
shr.u r18=r19,16 // get byte size of existing "dirty" partition
@@ -954,6 +1018,7 @@
mov r16=ar.bsp // get existing backing store pointer
LOAD_PHYS_STACK_REG_SIZE(r17)
(pKStk) br.cond.dpnt skip_rbs_switch
+#endif
/*
* Restore user backing store.
@@ -995,28 +1060,6 @@
shladd in0=loc1,3,r17
mov in1=0
;;
-#ifdef CONFIG_VIRT_CPU_ACCOUNTING
-account_sys_leave:
- // The size of current frame
- // (in * 2 + loc * N (N = 8 or 12) + out * 2)
- // is enough to work, so just take care to keep in0,in1
- mov loc0=ar.itc
- mov loc1=IA64_KR(CURRENT) // M2 (12 cycle read latency)
- ;;
- add loc2=TI_AC_STAMP+IA64_TASK_SIZE,loc1
- add loc3=TI_AC_STIME+IA64_TASK_SIZE,loc1
- ;;
- ld8 loc4=[loc2] // get last stamp
- ld8 loc5=[loc3] // cumulated stime
- ;;
- sub loc4=loc0,loc4 // elapsed time
- ;;
- add loc5=loc5,loc4 // sum
- ;;
- st8 [loc2]=loc0 // update stamp
- st8 [loc3]=loc5 // update stime
- ;;
-#endif
TEXT_ALIGN(32)
rse_clear_invalid:
#ifdef CONFIG_ITANIUM
Index: linux-2.6.24-rc5/arch/ia64/kernel/fsys.S
===================================================================
--- linux-2.6.24-rc5.orig/arch/ia64/kernel/fsys.S
+++ linux-2.6.24-rc5/arch/ia64/kernel/fsys.S
@@ -689,17 +689,23 @@
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
// mov.m r30=ar.itc is called in advance
add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2
- add r17=TI_AC_UTIME+IA64_TASK_SIZE,r2
+ add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2
;;
- ld8 r18=[r16] // get last stamp
- ld8 r19=[r17] // cumulated utime
+ ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // time at last check in kernel
+ ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // time at leave kernel
;;
- sub r18=r30,r18 // elapsed time
+ ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // cumulated stime
+ ld8 r21=[r17] // cumulated utime
+ sub r22=r19,r18 // stime before leave kernel
;;
- add r19=r19,r18 // sum
+ st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // update stamp
+ sub r18=r30,r19 // elapsed time in user mode
;;
- st8 [r16]=r30 // update stamp
- st8 [r17]=r19 // update utime
+ add r20=r20,r22 // sum stime
+ add r21=r21,r18 // sum utime
+ ;;
+ st8 [r16]=r20 // update stime
+ st8 [r17]=r21 // update utime
;;
#endif
mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
Index: linux-2.6.24-rc5/arch/ia64/kernel/ivt.S
===================================================================
--- linux-2.6.24-rc5.orig/arch/ia64/kernel/ivt.S
+++ linux-2.6.24-rc5/arch/ia64/kernel/ivt.S
@@ -841,18 +841,24 @@
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
// mov.m r30=ar.itc is called in advance, and r13 is current
add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13 // A
- add r17=TI_AC_UTIME+IA64_TASK_SIZE,r13 // A
+ add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13 // A
(pKStk) br.cond.spnt .skip_accounting // B unlikely skip
;;
- ld8 r18=[r16] // M get last stamp
- ld8 r19=[r17] // M cumulated utime
+ ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // M get last stamp
+ ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // M time at leave
;;
- sub r18=r30,r18 // A elapsed time
+ ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // M cumulated stime
+ ld8 r21=[r17] // M cumulated utime
+ sub r22=r19,r18 // A stime before leave
;;
- add r19=r19,r18 // A sum
+ st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // M update stamp
+ sub r18=r30,r19 // A elapsed time in user
;;
- st8 [r16]=r30 // M update stamp
- st8 [r17]=r19 // M update utime
+ add r20=r20,r22 // A sum stime
+ add r21=r21,r18 // A sum utime
+ ;;
+ st8 [r16]=r20 // M update stime
+ st8 [r17]=r21 // M update utime
;;
.skip_accounting:
#endif
@@ -1131,18 +1137,24 @@
ENTRY(account_sys_enter)
// mov.m r20=ar.itc is called in advance, and r13 is current
add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13
- add r17=TI_AC_UTIME+IA64_TASK_SIZE,r13
+ add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13
+ ;;
+ ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // time at last check in kernel
+ ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // time at left from kernel
+ ;;
+ ld8 r23=[r16],TI_AC_STAMP-TI_AC_STIME // cumulated stime
+ ld8 r21=[r17] // cumulated utime
+ sub r22=r19,r18 // stime before leave kernel
;;
- ld8 r18=[r16] // get last stamp
- ld8 r19=[r17] // cumulated utime
+ st8 [r16]=r20,TI_AC_STIME-TI_AC_STAMP // update stamp
+ sub r18=r20,r19 // elapsed time in user mode
;;
- sub r18=r20,r18 // elapsed time
+ add r23=r23,r22 // sum stime
+ add r21=r21,r18 // sum utime
;;
- add r19=r19,r18 // sum
+ st8 [r16]=r23 // update stime
+ st8 [r17]=r21 // update utime
;;
- st8 [r16]=r20 // update stamp
- st8 [r17]=r19 // update utime
- ;;
br.ret.sptk.many rp
END(account_sys_enter)
#endif
Index: linux-2.6.24-rc5/include/asm-ia64/thread_info.h
===================================================================
--- linux-2.6.24-rc5.orig/include/asm-ia64/thread_info.h
+++ linux-2.6.24-rc5/include/asm-ia64/thread_info.h
@@ -33,6 +33,7 @@
struct restart_block restart_block;
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
__u64 ac_stamp;
+ __u64 ac_leave;
__u64 ac_stime;
__u64 ac_utime;
#endif
-
To unsubscribe from this list: send the line "unsubscribe linux-ia64" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
|