Hi,
We use hardware breakpoints in CRIU and we found that sometimes we set
a break-point, but a process doesn't stop on it.
I write a small reproducer for this bug. It create two processes,
where a parent process traces a child. The parent process sets a
break-point and each time when the child stop on it, the parent sets
the variable "xxx" to A in a child process. The child runs an infinite
loop, where it check the variable "xxx" and sets it to B. If a child
process finds that xxx is equal to B, it exits with a non-zero code,
what means that a break-point was not triggered. The source code is
attached.
The reproducer uses a different break-point address if it is executed
with arguments than when it executed without arguments.
Then I made a few experiments. The bug is triggered, if we execute
this program a few times in a KVM virtual machine.
[root@fc22-vm ptrace]# ( while :; do ./ptrace_breakpoint > /dev/null
|| { echo "FAIL - $?"; break; }; done ) &
[3] 4088
[root@fc22-vm ptrace]# ( while :; do ./ptrace_breakpoint > /dev/null
|| { echo "FAIL - $?"; break; }; done ) &
[4] 4091
[root@fc22-vm ptrace]# ( while :; do ./ptrace_breakpoint 1 2 >
/dev/null || { echo "FAIL - $?"; break; }; done ) &
[5] 4094
[root@fc22-vm ptrace]# ( while :; do ./ptrace_breakpoint 1 2 >
/dev/null || { echo "FAIL - $?"; break; }; done ) &
[6] 4097
[8] 4103
[root@fc22-vm ptrace]# 0087: exit - 5
0131: exited, status=1
0126: wait: No child processes
FAIL - 3
I tried to execute the reproducer on the host (where kvm VM-s are
running), but the bug was not triggered during one hour.
When I executed the reproducer in VM without stopping processes on the
host, I found that a bug is triggered much faster in this case.
[root@fc22-vm ptrace]# ./ptrace_breakpoint 1
....
stop 24675
cont
child2 1
stop 24676
cont
child2 1
child2 5
0088: exit - 5
stop 24677
0132: exited, status=1
cont
0127: wait: No child processes
I know that this bug can be reproduced starting with the 4.2 kernel. I
haven't test older versions of the kernel.
I tried to print drX registers after a break-point. Looks like they
are set correctly.
Maybe someone has any ideas where a problem is or how it can be investigated.
Here is a criu issue for this problem:
https://github.com/xemul/criu/issues/107
Thanks,
Andrew
#define _GNU_SOURCE /* See feature_test_macros(7) */
#include <sys/syscall.h> /* For SYS_xxx definitions */
#include <unistd.h>
#include <signal.h>
#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/ptrace.h>
#include <errno.h>
#include <sys/user.h>
#include <asm/debugreg.h>
#include <string.h>
/* Copied from the gdb header gdb/nat/x86-dregs.h */
/* Debug registers' indices. */
#define DR_FIRSTADDR 0
#define DR_LASTADDR 3
#define DR_NADDR 4 /* The number of debug address registers. */
#define DR_STATUS 6 /* Index of debug status register (DR6). */
#define DR_CONTROL 7 /* Index of debug control register (DR7). */
#define DR_LOCAL_ENABLE_SHIFT 0 /* Extra shift to the local enable bit. */
#define DR_GLOBAL_ENABLE_SHIFT 1 /* Extra shift to the global enable bit. */
#define DR_ENABLE_SIZE 2 /* Two enable bits per debug register. */
/* Locally enable the break/watchpoint in the I'th debug register. */
#define X86_DR_LOCAL_ENABLE(i) (1 << (DR_LOCAL_ENABLE_SHIFT + DR_ENABLE_SIZE * (i)))
# define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
#define pr_perror(fmt, ...) \
fprintf(stderr, "%04d: " fmt ": %s\n", __LINE__, ##__VA_ARGS__, strerror(errno))
#define pr_err(fmt, ...) \
fprintf(stderr, "%04d: " fmt "\n", __LINE__, ##__VA_ARGS__)
int ptrace_set_breakpoint(pid_t pid, void *addr)
{
int ret;
/* Set a breakpoint */
if (ptrace(PTRACE_POKEUSER, pid,
offsetof(struct user, u_debugreg[DR_FIRSTADDR]),
addr)) {
pr_perror("Unable to setup a breakpoint into %d", pid);
return -1;
}
/* Enable the breakpoint */
if (ptrace(PTRACE_POKEUSER, pid,
offsetof(struct user, u_debugreg[DR_CONTROL]),
X86_DR_LOCAL_ENABLE(DR_FIRSTADDR))) {
pr_perror("Unable to enable the breakpoint for %d", pid);
return -1;
}
ret = ptrace(PTRACE_CONT, pid, NULL, NULL);
if (ret) {
pr_perror("Unable to restart the stopped tracee process %d", pid);
return -1;
}
return 1;
}
static long xxx = -1;
int child()
{
printf("child %d\n", xxx);
syscall(__NR_getppid);
if (xxx == 5) {
pr_err("exit - %d", xxx);
exit(1);
}
if (xxx > 0)
xxx = 5;
return 0;
}
int child2()
{
printf("child2 %d\n", xxx);
syscall(__NR_getppid);
if (xxx == 5) {
pr_err("exit - %d", xxx);
exit(1);
}
if (xxx > 0)
xxx = 5;
return 0;
}
int main(int argc, char **argv)
{
pid_t pid;
int status, i = 0;
int (*addr)();
if (argc > 1)
addr = child2;
else
addr = child;
pid = fork();
if (pid == 0) {
while (1)
addr();
}
if (ptrace(PTRACE_ATTACH, pid, NULL, NULL)) {
pr_perror("ptrace");
return 1;
}
if (waitpid(pid, &status, 0) == -1) {
pr_perror("waitpid");
return 1;
}
if (ptrace_set_breakpoint(pid, addr) != 1)
return 2;
while (1) {
if (waitpid(pid, &status, 0) != pid) {
pr_perror("wait");
return 3;
}
printf("stop %d\n", i++);
if (WIFEXITED(status)) {
pr_err("exited, status=%d", WEXITSTATUS(status));
} else if (WIFSIGNALED(status)) {
pr_err("killed by signal %d", WTERMSIG(status));
}
ptrace(PTRACE_POKEDATA, pid, &xxx, 1);
printf("cont\n");
ptrace(PTRACE_CONT, pid, NULL, NULL);
}
return 0;
}
