This is 1 of 3 patches for ia64 clocksource.
As john stultz wrote:
> So w/ x86_64, we've split the xtime_lock and get vgtod_lock, so that
> only when the vsyscall page is being updated do we hold a write on the
> vgtod_lock. This is safe as the vsyscall gtod does not access the
> kernel's time structures (xtime and friends). Instead it reads its copy
> of them that is made in update_vsyscall().
>
> So it should be fine to use the gtod_lock.sequence, assuming you're also
> not touching the kernel's xtime directly (and instead using copy of
> xtime made in update_vsyscall).
having copy of xtime is good solution.
This patch does:
- add wall_time/monotonic_time member to struct fsys_gtod_data.
- copy xtime to wall_time in update_vsyscall.
- calculate monotonic time in update_vsyscall, instead of doing
it in each gettimeofday call. And save it to monotonic_time.
- force loading gtod_lock.sequence before touching protected data.
- modify comments.
Then, the results of my test:
> # clocksource (Bob's) : default
> CPU 0: 4.66 (usecs) (0 errors / 2145680 iterations)
> CPU 1: 4.65 (usecs) (493 errors / 2148438 iterations)
> CPU 2: 4.63 (usecs) (668 errors / 2159461 iterations)
> CPU 3: 4.62 (usecs) (654 errors / 2163997 iterations)
> # clocksource (Bob's) : nojitter
> CPU 0: 0.14 (usecs) (0 errors / 70945550 iterations)
> CPU 1: 0.14 (usecs) (470 errors / 71640889 iterations)
> CPU 2: 0.14 (usecs) (664 errors / 70960917 iterations)
> CPU 3: 0.14 (usecs) (571 errors / 70956121 iterations)
> # clocksource (Bob's) : nolwsys
> CPU 0: 2.88 (usecs) (0 errors / 3475147 iterations)
> CPU 1: 2.88 (usecs) (0 errors / 3474881 iterations)
> CPU 2: 2.96 (usecs) (0 errors / 3382229 iterations)
> CPU 3: 2.97 (usecs) (0 errors / 3371004 iterations)
are now
> # savextime : default
> CPU 0: 5.49 (usecs) (0 errors / 1822461 iterations)
> CPU 1: 5.46 (usecs) (0 errors / 1830527 iterations)
> CPU 2: 5.45 (usecs) (0 errors / 1834697 iterations)
> CPU 3: 5.46 (usecs) (0 errors / 1831464 iterations)
> # savextime : nojitter
> CPU 0: 0.14 (usecs) (0 errors / 70918167 iterations)
> CPU 1: 0.14 (usecs) (0 errors / 71620242 iterations)
> CPU 2: 0.14 (usecs) (0 errors / 70946394 iterations)
> CPU 3: 0.14 (usecs) (0 errors / 70933043 iterations)
> # savextime : nolwsys
> CPU 0: 2.89 (usecs) (0 errors / 3459498 iterations)
> CPU 1: 2.89 (usecs) (0 errors / 3458876 iterations)
> CPU 2: 2.96 (usecs) (0 errors / 3377799 iterations)
> CPU 3: 2.97 (usecs) (0 errors / 3372353 iterations)
so errors are vanished by proper locking, but something in
"default" is still wrong.
(Continue to next patch.)
Thanks,
H.Seto
Signed-off-by: Hidetoshi Seto <seto.hidetoshi@xxxxxxxxxxxxxx>
-----
arch/ia64/kernel/asm-offsets.c | 28 +++++++++----
arch/ia64/kernel/fsys.S | 71 +++++++++++++++++-----------------
arch/ia64/kernel/fsyscall_gtod_data.h | 2
arch/ia64/kernel/time.c | 14 ++++++
4 files changed, 73 insertions(+), 42 deletions(-)
Index: linux-2.6.22/arch/ia64/kernel/asm-offsets.c
===================================================================
--- linux-2.6.22.orig/arch/ia64/kernel/asm-offsets.c
+++ linux-2.6.22/arch/ia64/kernel/asm-offsets.c
@@ -258,12 +258,24 @@
BLANK();
/* used by fsys_gettimeofday in arch/ia64/kernel/fsys.S */
- DEFINE(IA64_GTOD_LOCK_OFFSET, offsetof (struct fsyscall_gtod_data_t, lock));
- DEFINE(IA64_CLKSRC_MASK_OFFSET, offsetof (struct fsyscall_gtod_data_t, clk_mask));
- DEFINE(IA64_CLKSRC_MULT_OFFSET, offsetof (struct fsyscall_gtod_data_t, clk_mult));
- DEFINE(IA64_CLKSRC_SHIFT_OFFSET, offsetof (struct fsyscall_gtod_data_t, clk_shift));
- DEFINE(IA64_CLKSRC_MMIO_OFFSET, offsetof (struct fsyscall_gtod_data_t, clk_fsys_mmio));
- DEFINE(IA64_CLKSRC_CYCLE_LAST_OFFSET, offsetof (struct fsyscall_gtod_data_t, clk_cycle_last));
- DEFINE(IA64_ITC_LASTCYCLE_OFFSET, offsetof (struct fsyscall_gtod_data_t, itc_lastcycle));
- DEFINE(IA64_ITC_JITTER_OFFSET, offsetof (struct fsyscall_gtod_data_t, itc_jitter));
+ DEFINE(IA64_GTOD_LOCK_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, lock));
+ DEFINE(IA64_GTOD_WALL_TIME_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, wall_time));
+ DEFINE(IA64_GTOD_MONO_TIME_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, monotonic_time));
+ DEFINE(IA64_CLKSRC_MASK_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, clk_mask));
+ DEFINE(IA64_CLKSRC_MULT_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, clk_mult));
+ DEFINE(IA64_CLKSRC_SHIFT_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, clk_shift));
+ DEFINE(IA64_CLKSRC_MMIO_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, clk_fsys_mmio));
+ DEFINE(IA64_CLKSRC_CYCLE_LAST_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, clk_cycle_last));
+ DEFINE(IA64_ITC_LASTCYCLE_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, itc_lastcycle));
+ DEFINE(IA64_ITC_JITTER_OFFSET,
+ offsetof (struct fsyscall_gtod_data_t, itc_jitter));
}
Index: linux-2.6.22/arch/ia64/kernel/fsyscall_gtod_data.h
===================================================================
--- linux-2.6.22.orig/arch/ia64/kernel/fsyscall_gtod_data.h
+++ linux-2.6.22/arch/ia64/kernel/fsyscall_gtod_data.h
@@ -7,6 +7,8 @@
struct fsyscall_gtod_data_t {
seqlock_t lock;
+ struct timespec wall_time;
+ struct timespec monotonic_time;
cycle_t clk_mask;
u32 clk_mult;
u32 clk_shift;
Index: linux-2.6.22/arch/ia64/kernel/time.c
===================================================================
--- linux-2.6.22.orig/arch/ia64/kernel/time.c
+++ linux-2.6.22/arch/ia64/kernel/time.c
@@ -373,6 +373,20 @@
fsyscall_gtod_data.clk_fsys_mmio = c->fsys_mmio;
fsyscall_gtod_data.clk_cycle_last = c->cycle_last;
+ /* copy kernel time structures */
+ fsyscall_gtod_data.wall_time.tv_sec = wall->tv_sec;
+ fsyscall_gtod_data.wall_time.tv_nsec = wall->tv_nsec;
+ fsyscall_gtod_data.monotonic_time.tv_sec = wall_to_monotonic.tv_sec
+ + wall->tv_sec;
+ fsyscall_gtod_data.monotonic_time.tv_nsec = wall_to_monotonic.tv_nsec
+ + wall->tv_nsec;
+
+ /* normalize */
+ while (fsyscall_gtod_data.monotonic_time.tv_nsec >= NSEC_PER_SEC) {
+ fsyscall_gtod_data.monotonic_time.tv_nsec -= NSEC_PER_SEC;
+ fsyscall_gtod_data.monotonic_time.tv_sec++;
+ }
+
write_sequnlock_irqrestore(&fsyscall_gtod_data.lock, flags);
}
Index: linux-2.6.22/arch/ia64/kernel/fsys.S
===================================================================
--- linux-2.6.22.orig/arch/ia64/kernel/fsys.S
+++ linux-2.6.22/arch/ia64/kernel/fsys.S
@@ -178,18 +178,19 @@
// r14 = address of mask / mask value
// r15 = preserved: system call number
// r16 = preserved: current task pointer
- // r17 = wall to monotonic use
+ // r17 = (not used)
+ // r18 = (not used)
// r19 = address of itc_lastcycle
- // r20 = struct fsyscall_gtod_data / address of first element
+ // r20 = struct fsyscall_gtod_data (= address of gtod_lock.sequence)
// r21 = address of mmio_ptr
- // r22 = address of wall_to_monotonic
- // r23 = address of shift/ value
+ // r22 = address of wall_time or monotonic_time
+ // r23 = address of shift / value
// r24 = address mult factor / cycle_last value
// r25 = itc_lastcycle value
// r26 = address clocksource cycle_last
- // r27 = pointer to xtime
+ // r27 = (not used)
// r28 = sequence number at the beginning of critcal section
- // r29 = address of seqlock
+ // r29 = address of itc_jitter
// r30 = time processing flags / memory address
// r31 = pointer to result
// Predicates
@@ -203,30 +204,36 @@
// p14 = Divide by 1000
// p15 = Add monotonic
//
- // Note that instructions are optimized for McKinley. McKinley can process two
- // bundles simultaneously and therefore we continuously try to feed the CPU
- // two bundles and then a stop.
- tnat.nz p6,p0 = r31 // branch deferred since it does not fit into bundle structure
+ // Note that instructions are optimized for McKinley. McKinley can
+ // process two bundles simultaneously and therefore we continuously
+ // try to feed the CPU two bundles and then a stop.
+ //
+ // Additional note that code has changed a lot. Optimization is TBD.
+ // Comments begin with "?" are maybe outdated.
+ tnat.nz p6,p0 = r31 // ? branch deferred to fit later bundle
mov pr = r30,0xc000 // Set predicates according to function
add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
movl r20 = fsyscall_gtod_data // load fsyscall gettimeofday data address
;;
add r29 = IA64_ITC_JITTER_OFFSET,r20
- movl r27 = xtime
+ add r22 = IA64_GTOD_WALL_TIME_OFFSET,r20 // wall_time
ld4 r2 = [r2] // process work pending flags
-(p15) movl r22 = wall_to_monotonic
;;
+(p15) add r22 = IA64_GTOD_MONO_TIME_OFFSET,r20 // monotonic_time
add r21 = IA64_CLKSRC_MMIO_OFFSET,r20
add r19 = IA64_ITC_LASTCYCLE_OFFSET,r20
and r2 = TIF_ALLWORK_MASK,r2
-(p6) br.cond.spnt.few .fail_einval // deferred branch
+(p6) br.cond.spnt.few .fail_einval // ? deferred branch
;;
add r26 = IA64_CLKSRC_CYCLE_LAST_OFFSET,r20 // clksrc_cycle_last
cmp.ne p6, p0 = 0, r2 // Fallback if work is scheduled
(p6) br.cond.spnt.many fsys_fallback_syscall
;; // get lock.seq here new code, outer loop2!
.time_redo:
- ld4.acq r28 = [r20] // gtod_lock.sequence, Must be first in struct
+ ld4.acq r28 = [r20] // gtod_lock.sequence, Must take first
+ ;;
+ and r28 = ~1,r28 // And make sequence even to force retry if odd
+ ;;
ld8 r30 = [r21] // clocksource->mmio_ptr
add r24 = IA64_CLKSRC_MULT_OFFSET,r20
ld4 r2 = [r29] // itc_jitter value
@@ -235,7 +242,7 @@
;;
ld4 r3 = [r24] // clocksource mult value
ld8 r14 = [r14] // clocksource mask value
- cmp.eq p8,p9 = 0,r30 // Check for cpu timer, no mmio_ptr, set p8, clear p9
+ cmp.eq p8,p9 = 0,r30 // use cpu timer if no mmio_ptr
;;
setf.sig f7 = r3 // Setup for mult scaling of counter
(p8) cmp.ne p13,p0 = r2,r0 // need itc_jitter compensation, set p13
@@ -246,17 +253,16 @@
.cmpxchg_redo:
.pred.rel.mutex p8,p9
(p8) mov r2 = ar.itc // CPU_TIMER. 36 clocks latency!!!
-(p9) ld8 r2 = [r30] // readq(ti->address). Could also have latency issues..
+(p9) ld8 r2 = [r30] // MMIO_TIMER. Could also have latency issues..
(p13) ld8 r25 = [r19] // get itc_lastcycle value
- ;; // could be removed by moving the last add upward
- ld8 r9 = [r27],IA64_TIMESPEC_TV_NSEC_OFFSET
-(p15) ld8 r17 = [r22],IA64_TIMESPEC_TV_NSEC_OFFSET
+ ;; // ? could be removed by moving the last add upward
+ ld8 r9 = [r22],IA64_TIMESPEC_TV_NSEC_OFFSET // tv_sec
;;
- ld8 r8 = [r27],-IA64_TIMESPEC_TV_NSEC_OFFSET // xtime.tv_nsec
+ ld8 r8 = [r22],-IA64_TIMESPEC_TV_NSEC_OFFSET // tv_nsec
(p13) sub r3 = r25,r2 // Diff needed before comparison (thanks davidm)
;;
(p13) cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
- sub r10 = r2,r24 // current_counter - last_counter
+ sub r10 = r2,r24 // current_cycle - last_cycle
;;
(p6) sub r10 = r25,r24 // time we got was less than last_cycle
(p7) mov ar.ccv = r25 // more than last_cycle. Prep for cmpxchg
@@ -267,23 +273,20 @@
nop.i 123
;;
(p7) cmpxchg8.rel r3 = [r19],r2,ar.ccv
-EX(.fail_efault, probe.w.fault r31, 3) // This takes 5 cycles and we have spare time
+ // fault check takes 5 cycles and we have spare time
+EX(.fail_efault, probe.w.fault r31, 3)
xmpy.l f8 = f8,f7 // nsec_per_cyc*(counter-last_counter)
-(p15) add r9 = r9,r17 // Add wall to monotonic.secs to result secs
;;
// End cmpxchg critical section loop1
-(p15) ld8 r17 = [r22],-IA64_TIMESPEC_TV_NSEC_OFFSET
(p7) cmp.ne p7,p0 = r25,r3 // if cmpxchg not successful redo
(p7) br.cond.dpnt.few .cmpxchg_redo // inner loop1
- // simulate tbit.nz.or p7,p0 = r28,0
- and r28 = ~1,r28 // Make sequence even to force retry if odd
+ // ? simulate tbit.nz.or p7,p0 = r28,0
getf.sig r2 = f8
mf
;;
- ld4 r10 = [r20] // gtod_lock.sequence, old xtime_lock.sequence
-(p15) add r8 = r8, r17 // Add monotonic.nsecs to nsecs
+ ld4 r10 = [r20] // gtod_lock.sequence
shr.u r2 = r2,r23 // shift by factor
- ;; // overloaded 3 bundles!
+ ;; // ? overloaded 3 bundles!
// End critical section.
add r8 = r8,r2 // Add xtime.nsecs
cmp4.ne.or p7,p0 = r28,r10
@@ -297,19 +300,19 @@
.time_normalize:
mov r21 = r8
cmp.ge p6,p0 = r8,r2
-(p14) shr.u r20 = r8, 3 // We can repeat this if necessary just wasting some time
+(p14) shr.u r20 = r8, 3 // We can repeat this if necessary just wasting time
;;
(p14) setf.sig f8 = r20
(p6) sub r8 = r8,r2
-(p6) add r9 = 1,r9 // two nops before the branch.
-(p14) setf.sig f7 = r3 // Chances for repeats are 1 in 10000 for gettod
+(p6) add r9 = 1,r9 // two nops before the branch.
+(p14) setf.sig f7 = r3 // Chances for repeats are 1 in 10000 for gettod
(p6) br.cond.dpnt.few .time_normalize
;;
// Divided by 8 though shift. Now divide by 125
// The compiler was able to do that with a multiply
// and a shift and we do the same
-EX(.fail_efault, probe.w.fault r23, 3) // This also costs 5 cycles
-(p14) xmpy.hu f8 = f8, f7 // xmpy has 5 cycles latency so use it...
+EX(.fail_efault, probe.w.fault r23, 3) // This also costs 5 cycles
+(p14) xmpy.hu f8 = f8, f7 // xmpy has 5 cycles latency so use it
;;
mov r8 = r0
(p14) getf.sig r2 = f8
-
To unsubscribe from this list: send the line "unsubscribe linux-ia64" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
|