I’m attempting to add DMA support to the ti_am335x_adc driver but when running the Lauterback debugger, I get strange timer corruption and the kernel completely loops forever in include/linux/math64.h and takes the “bhi” branch below and never gets to C00BB2A0.
============
|static___always_inline_u32
|__iter_div_u64_rem(u64_dividend,_u32_divisor,_u64_*remainder)
|{
|________u32_ret_=_0;
|
122 |________while_(dividend_>=_divisor)_{
NSR:0000:C00BB294 |E1590007____________cmp_____r9,r7
NSR:0000:C00BB298 |01580006____________cmpeq___r8,r6
NSR:0000:C00BB29C |8AFFFFF7____________bhi_____0xC00BB280
NSR:0000:C00BB2A0 |E1A02008____________cpy_____r2,r8
NSR:0000:C00BB2A4 |E1A0000C____________cpy_____r0,r12___________;_r0,ret
NSR:0000:C00BB2A8 |E3A01000____________mov_____r1,#0x0__________;_r1,#0
NSR:0000:C00BB2AC |EA000001____________b_______0xC00BB2B8
NSR:0000:C00BB2B0 |E3A00000____________mov_____r0,#0x0__________;_r0,#0
NSR:0000:C00BB2B4 |E3A01000____________mov_____r1,#0x0__________;_r1,#0
|...
|_*_@a:__________pointer_to_timespec64_to_be_incremented
=============
After googling, I came across this article:
https://forums.xilinx.com/t5/Embedded-Linux/Linux-hangs-2min-on-CPU0-if-bare-metal-CPU1-is-restarted-with/td-p/561337?db=5
Which explains the behavior I am seeing. Now my situation is not exactly the same, because I am running on a BeagleBoneBlack so I’m only running on a single processor.
However, this did explained the behavior I was seeing, so I modified timekeeping_get_ns and __getnstimeofday64 in kernel/time/timekeeping.c as follows:
=============
static inline s64 timekeeping_get_ns(struct tk_read_base *tkr)
{
cycle_t delta, cycle_now;
s64 nsec;
/* read clocksource */
cycle_now = tkr->read(tkr->clock);
delta = timekeeping_get_delta(tkr);
if(delta < 1) {
printk_deferred(KERN_ERR "timekeeping_get_ns: cycle_now = 0x%llx", cycle_now );
printk_deferred(KERN_ERR "timekeeping_get_ns: tkr->cycle_last = 0x%llx", tkr->cycle_last );
printk_deferred(KERN_ERR "timekeeping_get_ns: tkr->mask = 0x%llx", tkr->mask );
printk_deferred(KERN_ERR "timekeeping_get_ns: tkr->mult = 0x%x", tkr->mult );
printk_deferred(KERN_ERR "timekeeping_get_ns: tkr->xtime_nsec = 0x%llx", tkr->xtime_nsec );
printk_deferred(KERN_ERR "timekeeping_get_ns: tkr->shift = 0x%x", tkr->shift );
printk_deferred(KERN_ERR "timekeeping_get_ns: delta = 0x%llx", delta );
BUG_ON(1);
}
nsec = delta * tkr->mult + tkr->xtime_nsec;
nsec >>= tkr->shift;
/* If arch requires, add in get_arch_timeoffset() */
return nsec + arch_gettimeoffset();
}
int __getnstimeofday64(struct timespec64 *ts)
{
struct timekeeper *tk = &tk_core.timekeeper;
unsigned long seq;
s64 nsecs = 0;
do {
seq = read_seqcount_begin(&tk_core.seq);
ts->tv_sec = tk->xtime_sec;
nsecs = timekeeping_get_ns(&tk->tkr_mono);
} while (read_seqcount_retry(&tk_core.seq, seq));
if( nsecs < 0 )
{
printk_deferred(KERN_ERR "__getnstimeofday64: nsecs = 0x%llx", nsecs );
printk_deferred(KERN_ERR "__getnstimeofday64: ts->tv_nsec = 0x%lx", ts->tv_nsec );
printk_deferred(KERN_ERR "__getnstimeofday64: ts->tv_sec = 0x%llx", ts->tv_sec );
BUG_ON(1);
}
ts->tv_nsec = 0;
timespec64_add_ns(ts, nsecs);
/*
* Do not bail out early, in case there were callers still using
* the value, even in the face of the WARN_ON.
*/
if (unlikely(timekeeping_suspended))
return -EAGAIN;
return 0;
}
=============
So here is what I see after single stepping and running and stopping several times. As you can see, the timer is negative and this is causing a lockup. I have seen the same occur with timekeeping_get_ns where delta is negative ( tkr->cycle_last occurs after cycle_now) . Seems like a race condition. Does anyone have any idea as to why this would be happening, or alternatively, how can I just cleanup this up when I get a negative value. I tried setting tkr->cycle_last = cycle_now, but that didn’t work.
Regards,
John
=============
[ 2058.669928] __getnstimeofday64: nsecs = 0xffffffe4706f8f3f
[ 2058.669941] __getnstimeofday64: ts->tv_nsec = 0x0
[ 2058.669999] __getnstimeofday64: ts->tv_sec = 0x57219e56
[ 2058.670002] ------------[ cut here ]------------
[ 2058.690823] kernel BUG at kernel/time/timekeeping.c:684!
[ 2058.696188] Internal error: Oops - BUG: 0 [#1] PREEMPT SMP ARM
[ 2058.702075] Modules linked in: ti_am335x_adc industrialio_buffer_dmaengine industrialio_buffer_dma kfifo_buf ti_am335x_tsc ti_am335x_tscadc iio_trig_sysfs industrialio usb_f_acm u_serial usbc
[ 2058.736017] CPU: 0 PID: 230 Comm: avahi-daemon Not tainted 4.1.13-ti-r33 #105
[ 2058.743210] Hardware name: Generic AM33XX (Flattened Device Tree)
[ 2058.749361] task: dc9a54c0 ti: dc9a6000 task.ti: dc9a6000
[ 2058.754838] PC is at __getnstimeofday64+0x1c8/0x250
[ 2058.759775] LR is at preempt_count_sub+0xec/0x120
[ 2058.764532] pc : [<c00bb5e0>] lr : [<c006ffc8>] psr: 600d0013
[ 2058.764532] sp : dc9a7f18 ip : dc9a7ec8 fp : dc9a7f4c
[ 2058.776093] r10: 00000000 r9 : 00000000 r8 : df6057b6
[ 2058.781367] r7 : 91c1be3c r6 : fffffffa r5 : dc9a7f60 r4 : 00000006
[ 2058.787951] r3 : 00000000 r2 : 00000000 r1 : 00000000 r0 : 0000002c
[ 2058.794539] Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment user
[ 2058.801735] Control: 10c5387d Table: 9c9b0019 DAC: 00000015
[ 2058.807533] Process avahi-daemon (pid: 230, stack limit = 0xdc9a6218)
[ 2058.814030] Stack: (0xdc9a7f18 to 0xdc9a8000)
[ 2058.818438] 7f00: 80f36d5d 00000000
[ 2058.826695] 7f20: df6057b6 00000000 dc9a7f88 bee0cc18 00036418 0000004e c00108e4 dc9a6000
[ 2058.834950] 7f40: dc9a7f5c dc9a7f50 c00bb680 c00bb424 dc9a7f84 dc9a7f60 c00bb6c0 c00bb674
[ 2058.843206] 7f60: 57219e56 00000000 00000000 01a38f58 00000000 00000000 dc9a7fa4 dc9a7f88
[ 2058.851461] 7f80: c00b16e8 c00bb6a8 00000000 dc9a7f98 01a38f58 01a40cec 00000000 dc9a7fa8
[ 2058.859717] 7fa0: c0010700 c00b16c8 01a38f58 01a40cec bee0cc18 00000000 00000001 57219e76
[ 2058.867973] 7fc0: 01a38f58 01a40cec 00036418 0000004e 00000000 01a38f58 01a38fc8 00000000
[ 2058.876231] 7fe0: b6fa20b4 bee0cc14 b6f8ec91 b6e21fd6 600d0030 bee0cc18 9fdf2821 9fdf2c21
[ 2058.884517] [<c00bb5e0>] (__getnstimeofday64) from [<c00bb680>] (getnstimeofday64+0x18/0x34)
[ 2058.893044] [<c00bb680>] (getnstimeofday64) from [<c00bb6c0>] (do_gettimeofday+0x24/0x50)
[ 2058.901316] [<c00bb6c0>] (do_gettimeofday) from [<c00b16e8>] (SyS_gettimeofday+0x2c/0xc4)
[ 2058.909591] [<c00b16e8>] (SyS_gettimeofday) from [<c0010700>] (ret_fast_syscall+0x0/0x54)
[ 2058.917850] Code: e3010db8 e1c520d0 e34c00c4 eb23c870 (e7f001f2)
[ 2058.924017] ---[ end trace 0d6a017508ae57cb ]---
[ 2058.934532] __getnstimeofday64: nsecs = 0xffffffe480352172
[ 2058.934544] __getnstimeofday64: ts->tv_nsec = 0x1ba5058
[ 2058.934597] __getnstimeofday64: ts->tv_sec = 0x57219e56
[ 2058.934600] ------------[ cut here ]------------
[ 2058.955932] kernel BUG at kernel/time/timekeeping.c:684!
[ 2058.961296] Internal error: Oops - BUG: 0 [#2] PREEMPT SMP ARM
[ 2058.967181] Modules linked in: ti_am335x_adc industrialio_buffer_dmaengine industrialio_buffer_dma kfifo_buf ti_am335x_tsc ti_am335x_tscadc iio_trig_sysfs industrialio usb_f_acm u_serial usbc
[ 2059.001115] CPU: 0 PID: 112 Comm: systemd-journal Tainted: G D 4.1.13-ti-r33 #105
[ 2059.009792] Hardware name: Generic AM33XX (Flattened Device Tree)
[ 2059.015942] task: ddddc300 ti: dc812000 task.ti: dc812000
[ 2059.021413] PC is at __getnstimeofday64+0x1c8/0x250
[ 2059.026350] LR is at preempt_count_sub+0xec/0x120
[ 2059.031108] pc : [<c00bb5e0>] lr : [<c006ffc8>] psr: 600f0013
[ 2059.031108] sp : dc813f18 ip : dc813ec8 fp : dc813f4c
[ 2059.042667] r10: 00000000 r9 : 00000000 r8 : dfc13f8f
[ 2059.047943] r7 : 9200d485 r6 : fffffffa r5 : dc813f60 r4 : 00000006
[ 2059.054526] r3 : 00000000 r2 : 00000000 r1 : 00000000 r0 : 0000002c
[ 2059.061113] Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment user
[ 2059.068309] Control: 10c5387d Table: 9c818019 DAC: 00000015
[ 2059.074109] Process systemd-journal (pid: 112, stack limit = 0xdc812218)
[ 2059.080866] Stack: (0xdc813f18 to 0xdc814000)
[ 2059.085274] 3f00: 81545535 00000000
[ 2059.093530] 3f20: dfc13f8f 00000000 dc813f88 00000000 be9ef7b8 00000107 c00108e4 dc812000
[ 2059.101784] 3f40: dc813f5c dc813f50 c00bb680 c00bb424 dc813f84 dc813f60 c00b7908 c00bb674
[ 2059.110039] 3f60: 57219e56 00000000 01ba5058 01ba5070 01ba5058 be9ef784 dc813fa4 dc813f88
[ 2059.118293] 3f80: c00b88a4 c00b78f0 00000000 dc813f98 b6f95f10 be9ef784 00000000 dc813fa8
[ 2059.126548] 3fa0: c0010700 c00b8874 b6f95f10 be9ef784 00000000 be9ef784 00000300 46f53800
[ 2059.134802] 3fc0: b6f95f10 be9ef784 be9ef7b8 00000107 00000017 b6f95f10 00034028 0003012c
[ 2059.143058] 3fe0: 00000107 be9ef774 b6ee386f b6e5c8e6 800f0030 00000000 00000000 00000000
[ 2059.151336] [<c00bb5e0>] (__getnstimeofday64) from [<c00bb680>] (getnstimeofday64+0x18/0x34)
[ 2059.159877] [<c00bb680>] (getnstimeofday64) from [<c00b7908>] (posix_clock_realtime_get+0x24/0x3c)
[ 2059.168928] [<c00b7908>] (posix_clock_realtime_get) from [<c00b88a4>] (SyS_clock_gettime+0x3c/0x98)
[ 2059.178072] [<c00b88a4>] (SyS_clock_gettime) from [<c0010700>] (ret_fast_syscall+0x0/0x54)
[ 2059.186416] Code: e3010db8 e1c520d0 e34c00c4 eb23c870 (e7f001f2)
[ 2059.192581] ---[ end trace 0d6a017508ae57cc ]---
[ 2059.201543] __getnstimeofday64: nsecs = 0xffffffe4901f3ee5
[ 2059.201555] __getnstimeofday64: ts->tv_nsec = 0xbeeb9510
[ 2059.201606] __getnstimeofday64: ts->tv_sec = 0x57219e56
[ 2059.201608] ------------[ cut here ]------------
[ 2059.223023] kernel BUG at kernel/time/timekeeping.c:684!
[ 2059.228386] Internal error: Oops - BUG: 0 [#3] PREEMPT SMP ARM
[ 2059.234270] Modules linked in: ti_am335x_adc industrialio_buffer_dmaengine industrialio_buffer_dma kfifo_buf ti_am335x_tsc ti_am335x_tscadc iio_trig_sysfs industrialio usb_f_acm u_serial usbc
[ 2059.268193] CPU: 0 PID: 1 Comm: systemd Tainted: G D 4.1.13-ti-r33 #105
[ 2059.275996] Hardware name: Generic AM33XX (Flattened Device Tree)
[ 2059.282148] task: dda81480 ti: dda82000 task.ti: dda82000
[ 2059.287617] PC is at __getnstimeofday64+0x1c8/0x250
[ 2059.292553] LR is at preempt_count_sub+0xec/0x120
[ 2059.297310] pc : [<c00bb5e0>] lr : [<c006ffc8>] psr: 60000013
[ 2059.297310] sp : dda83f18 ip : dda83ec8 fp : dda83f4c
[ 2059.308871] r10: 00000000 r9 : 00000000 r8 : e02307f1
[ 2059.314146] r7 : 92407cfb r6 : fffffffa r5 : dda83f60 r4 : 00000006
[ 2059.320729] r3 : 00000000 r2 : 00000000 r1 : 00000000 r0 : 0000002c
[ 2059.327314] Flags: nZCv IRQs on FIQs on Mode SVC_32 ISA ARM Segment user
[ 2059.334510] Control: 10c5387d Table: 9de54019 DAC: 00000015
[ 2059.340309] Process systemd (pid: 1, stack limit = 0xdda82218)
[ 2059.346194] Stack: (0xdda83f18 to 0xdda84000)
[ 2059.350602] 3f00: 81b61d98 00000000
[ 2059.358858] 3f20: e02307f1 00000000 dda83f88 00000000 02000bc8 00000107 c00108e4 dda82000
[ 2059.367113] 3f40: dda83f5c dda83f50 c00bb680 c00bb424 dda83f84 dda83f60 c00b7908 c00bb674
[ 2059.375369] 3f60: 57219e56 00000000 beeb9510 02000be0 beeb9510 beeb94dc dda83fa4 dda83f88
[ 2059.383623] 3f80: c00b88a4 c00b78f0 00000000 dda83f98 b6f9df10 beeb94dc 00000000 dda83fa8
[ 2059.391878] 3fa0: c0010700 c00b8874 b6f9df10 beeb94dc 00000000 beeb94dc 0000052c b5476a00
[ 2059.400133] 3fc0: b6f9df10 beeb94dc 02000bc8 00000107 ffffffff ffffffff 00000004 000a5c90
[ 2059.408389] 3fe0: 00000107 beeb94cc b6eeb86f b6e648e6 80000030 00000000 04100126 04400102
[ 2059.416667] [<c00bb5e0>] (__getnstimeofday64) from [<c00bb680>] (getnstimeofday64+0x18/0x34)
[ 2059.425207] [<c00bb680>] (getnstimeofday64) from [<c00b7908>] (posix_clock_realtime_get+0x24/0x3c)
[ 2059.434258] [<c00b7908>] (posix_clock_realtime_get) from [<c00b88a4>] (SyS_clock_gettime+0x3c/0x98)
[ 2059.443403] [<c00b88a4>] (SyS_clock_gettime) from [<c0010700>] (ret_fast_syscall+0x0/0x54)
[ 2059.451751] Code: e3010db8 e1c520d0 e34c00c4 eb23c870 (e7f001f2)
[ 2059.457919] ---[ end trace 0d6a017508ae57cd ]---
[ 2059.468193] Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b
[ 2059.468193]
[ 2059.477447] drm_kms_helper: panic occurred, switching back to text console
[ 2059.484438] ---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b
[ 2059.484438]
Regards,
John
--
To unsubscribe from this list: send the line "unsubscribe linux-omap" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
