POSIX timer wakeups not preempting RT threads

[Martin Hicks <martin@xxxxxxxxxxxxxxx>]

Hi,

I'm just getting started with RT linux, and am porting an existing
threaded application from another RTOS.

I'm looking at using a POSIX timer to cause an action to be run every
500us, but I'm seeing that when the hrtimer fires while the CPU is
busy running another SCHED_RR/SCHED_FIFO process that the hrtimer is
deferred until the process releases the CPU.  I'm using SIGEV_SIGNAL
to deliver the timer expiry to a thread that's running at a higher RT
Priority than the other tasks, but no preemption seems to occur.

If the "other" tasks are running with SCHED_OTHER then the signal is
delivered immediately.

I've tried using 'cset shield -c 1' and without (just pinning use
taskset -c 1), and modified priorities and scheduler with chrt.
The test is running on x86_64 with 6.3.9-rt15. PREEMPT_RT is enabled.

A simple test that illustrates the idea is attached.
Is this supposed to work?

mh

#define _GNU_SOURCE
#include <stdio.h>
#include <pthread.h>
#include <signal.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <fcntl.h>
#include <stdlib.h>
#include <math.h>

#define TIMER_PRIORITY 10
#define WORKER_PRIORITY 9

#define ONE_KHZ (1000000L)
#define KHZ(n)  ONE_KHZ/n

#define WORKERS 2
volatile double worker_data[WORKERS];

#define WORK_INTERVAL 20  /* ms to work per sec */

double current_time_double(void)
{
    struct timespec ts;

    clock_gettime(CLOCK_MONOTONIC, &ts);
    return ts.tv_sec*1e6 + ts.tv_nsec/1000.0;
}

void *worker_func(void *arg)
{
  unsigned long num = (unsigned long)arg;
  char thread_name[16];
  int init_sleep = random() % 1000;
  int ret;

  snprintf(thread_name, 16, "worker_%ld", num);
  if ((ret = pthread_setname_np(pthread_self(), thread_name)) != 0) {
    printf("worker %ld: Failed setname: %s\n", num, strerror(ret));
    return NULL;
  }

  struct sched_param param = {.sched_priority = WORKER_PRIORITY};
  if (sched_setscheduler(0, SCHED_RR, &param) == -1) {
    printf("worker %ld: Failed sched_setscheduler(): %s\n", num, strerror(errno));
    return NULL;
  }

  worker_data[num] = init_sleep;
  printf("Worker %ld started. Init sleep %d\n", num, init_sleep);
  usleep(init_sleep*1000);

  for (;;) {
    double work_start = current_time_double();
    int i;

    do {
      for (i = 0; i < 100000; i++)
        worker_data[num] *= M_PI;
    } while (current_time_double() - work_start < WORK_INTERVAL*1000);
    usleep((1000-WORK_INTERVAL)*1000);
  }
}



int timer_pipe[2];
#define TIMER_EXPIRY 0x55

/*
 * Invoked each time the itimer expires
 */
void timer_handler(int sig, siginfo_t *info, void *ucontext)
{
  char msg = TIMER_EXPIRY;
  int ret = write(timer_pipe[1], &msg, sizeof(msg));
  if (ret != 1)
    printf("pipe write failed: %s\n", strerror(errno));
}


void *create_timer_thread(void *arg)
{
  struct sigevent event;
  struct itimerspec itime;
  timer_t timer_id;
  sigset_t set;
  struct sigaction sa;
  int ret;
  double last_expiry;

  pthread_setname_np(pthread_self(), "timer_thread");

  struct sched_param param = {.sched_priority = TIMER_PRIORITY};
  if (sched_setscheduler(0, SCHED_RR, &param) == -1) {
    printf("timer_thread: Failed sched_setscheduler(): %s\n", strerror(errno));
    return NULL;
  }

  /* Set up pipe to communicate between timer handler and this function */
  if (pipe(timer_pipe))
    perror("pipe");

  event.sigev_notify = SIGEV_SIGNAL;
  event.sigev_signo = SIGRTMIN+1;
  event.sigev_notify_attributes = NULL;
  event.sigev_value.sival_ptr = &timer_id;

  if (timer_create(CLOCK_REALTIME, &event, &timer_id))
    perror("timer_create");

  /* Set up signal to be handled by this thread */
  sigemptyset(&set);
  sigaddset(&set, SIGRTMIN+1);
  if ((ret = pthread_sigmask(SIG_UNBLOCK, &set, NULL)) != 0)
    printf("Failed pthread_sigmask(): %s\n", strerror(ret));
  memset(&sa, 0, sizeof(sa));
  sa.sa_flags = SA_SIGINFO;
  sa.sa_sigaction = timer_handler;
  if (sigaction(SIGRTMIN+1, &sa, NULL) == -1)
    printf("Failed sigaction(): %s\n", strerror(errno));

  itime.it_value.tv_sec = 0;
  itime.it_value.tv_nsec = KHZ(2);
  itime.it_interval.tv_sec = 0;
  itime.it_interval.tv_nsec = KHZ(2);
  timer_settime(timer_id, 0, &itime, NULL);

  last_expiry = current_time_double();
  for (;;) {
    char msg;
    int bytes_read = read(timer_pipe[0], &msg, sizeof(msg));

    if (bytes_read == -1 && errno == EINTR)
        continue;
    else if (bytes_read != 1) {
      printf("Read error, bytes_read=%d (%s)\n", bytes_read, strerror(errno));
      exit(1);
    }
    if (msg == TIMER_EXPIRY) {
      double cur_time = current_time_double();
      if (cur_time - last_expiry > 1000)
        printf("delayed timer expiry %f\n", cur_time - last_expiry);
      last_expiry = cur_time;
    } else
      printf("Unknown message: %hhu\n", msg);
  }

}

int main(void)
{
  sigset_t set;
  int ret;
  pthread_t workers[WORKERS];
  pthread_t timer_tid;
  unsigned long i;

  srandom((int)current_time_double());

  /* Block SIGRTMIN+1 to all threads - unblock later in timer thread */
  sigemptyset(&set);
  sigaddset(&set, SIGRTMIN+1);
  if ((ret = pthread_sigmask(SIG_BLOCK, &set, NULL)) != 0)
    printf("Failed sigmask(): %s\n", strerror(ret));

  for (i = 0; i < WORKERS; i++)
    if ((ret = pthread_create(&workers[i], NULL, worker_func, (void *)i)) != 0) {
      errno = ret;
      perror("pthread_create(worker)");
    }

  /* Now create timer thread */
  if ((ret = pthread_create(&timer_tid, NULL, create_timer_thread, NULL)) != 0) {
    errno = ret;
    perror("pthread_create(timer)");
  }

  //sleep(30);
  pause();
  return 0;
}