x15/kern/perfmon.c

/*
 * Copyright (c) 2014-2018 Remy Noel.
 * Copyright (c) 2014-2018 Richard Braun.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *
 * Locking order :
 *
 *             thread_runq -+
 *                          |
 *   event -+-> interrupts -+-> td
 *          |
 *          +-> pmu
 *
 * TODO Kernel/user mode seggregation.
 */

#include <assert.h>
#include <errno.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>

#include <kern/clock.h>
#include <kern/init.h>
#include <kern/list.h>
#include <kern/log.h>
#include <kern/macros.h>
#include <kern/percpu.h>
#include <kern/perfmon.h>
#include <kern/perfmon_types.h>
#include <kern/spinlock.h>
#include <kern/syscnt.h>
#include <kern/thread.h>
#include <kern/timer.h>
#include <kern/xcall.h>
#include <machine/boot.h>
#include <machine/cpu.h>

/*
 * Minimum hardware counter poll interval, in milliseconds.
 *
 * The main purpose of polling hardware counters is to detect overflows
 * when the driver is unable to reliably use overflow interrupts.
 */
#define PERFMON_MIN_POLL_INTERVAL   50

// Internal event flags.
#define PERFMON_EF_TYPE_CPU         0x100
#define PERFMON_EF_ATTACHED         0x200
#define PERFMON_EF_PUBLIC_MASK   \
  (PERFMON_EF_KERN | PERFMON_EF_USER | PERFMON_EF_RAW)

/*
 * Per-CPU performance monitoring counter.
 *
 * When an event is attached to a processor, the matching per-CPU PMC get
 * referenced. When a per-CPU PMC is referenced, its underlying hardware
 * counter is active.
 *
 * Interrupts and preemption must be disabled on access.
 */
struct perfmon_cpu_pmc
{
  unsigned int nr_refs;
  unsigned int pmc_id;
  unsigned int raw_event_id;
  uint64_t raw_value;
  uint64_t value;
};

/*
 * Per-CPU performance monitoring unit.
 *
 * Per-CPU PMCs are indexed the same way as global PMCs.
 *
 * Interrupts and preemption must be disabled on access.
 */
struct perfmon_cpu_pmu
{
  struct perfmon_dev *dev;
  unsigned int cpu;
  struct perfmon_cpu_pmc pmcs[PERFMON_MAX_PMCS];
  struct timer poll_timer;
  struct syscnt sc_nr_overflows;
};

/*
 * Performance monitoring counter.
 *
 * When a PMC is used, it maps a raw event to a hardware counter.
 * A PMC is used if and only if its reference counter isn't zero.
 */
struct perfmon_pmc
{
  unsigned int nr_refs;
  unsigned int pmc_id;
  unsigned int raw_event_id;
};

/*
 * Performance monitoring unit.
 *
 * There is a single system-wide logical PMU, used to globally allocate
 * PMCs. Reserving a counter across the entire system ensures thread
 * migration isn't hindered by performance monitoring.
 *
 * Locking the global PMU is only required when allocating or releasing
 * a PMC. Once allocated, the PMC may safely be accessed without hodling
 * the lock.
 */
struct perfmon_pmu
{
  struct perfmon_dev *dev;
  struct spinlock lock;
  struct perfmon_pmc pmcs[PERFMON_MAX_PMCS];
};

static struct perfmon_pmu perfmon_pmu;
static struct perfmon_cpu_pmu perfmon_cpu_pmu __percpu;

static struct perfmon_pmu*
perfmon_get_pmu (void)
{
  return (&perfmon_pmu);
}

static struct perfmon_cpu_pmu*
perfmon_get_local_cpu_pmu (void)
{
  assert (!thread_preempt_enabled ());
  return (cpu_local_ptr (perfmon_cpu_pmu));
}

static struct perfmon_cpu_pmu*
perfmon_get_cpu_pmu (unsigned int cpu)
{
  return (percpu_ptr (perfmon_cpu_pmu, cpu));
}

static void __init
perfmon_pmc_init (struct perfmon_pmc *pmc)
{
  pmc->nr_refs = 0;
}

static bool
perfmon_pmc_used (const struct perfmon_pmc *pmc)
{
  return (pmc->nr_refs != 0);
}

static unsigned int
perfmon_pmc_id (const struct perfmon_pmc *pmc)
{
  return (pmc->pmc_id);
}

static unsigned int
perfmon_pmc_raw_event_id (const struct perfmon_pmc *pmc)
{
  return (pmc->raw_event_id);
}

static void
perfmon_pmc_use (struct perfmon_pmc *pmc, unsigned int pmc_id,
                 unsigned int raw_event_id)
{
  assert (!perfmon_pmc_used (pmc));
  pmc->nr_refs = 1;
  pmc->pmc_id = pmc_id;
  pmc->raw_event_id = raw_event_id;
}

static void
perfmon_pmc_ref (struct perfmon_pmc *pmc)
{
  assert (perfmon_pmc_used (pmc));
  ++pmc->nr_refs;
}

static void
perfmon_pmc_unref (struct perfmon_pmc *pmc)
{
  assert (perfmon_pmc_used (pmc));
  --pmc->nr_refs;
}

static unsigned int
perfmon_pmu_get_pmc_index (const struct perfmon_pmu *pmu,
                           const struct perfmon_pmc *pmc)
{
  size_t pmc_index = pmc - pmu->pmcs;
  assert (pmc_index < ARRAY_SIZE (pmu->pmcs));
  return (pmc_index);
}

static struct perfmon_pmc*
perfmon_pmu_get_pmc (struct perfmon_pmu *pmu, unsigned int index)
{
  assert (index < ARRAY_SIZE (pmu->pmcs));
  return (&pmu->pmcs[index]);
}

static void __init
perfmon_pmu_init (struct perfmon_pmu *pmu)
{
  pmu->dev = NULL;
  spinlock_init (&pmu->lock);

  for (size_t i = 0; i < ARRAY_SIZE (pmu->pmcs); i++)
    perfmon_pmc_init (perfmon_pmu_get_pmc (pmu, i));
}

static void __init
perfmon_pmu_set_dev (struct perfmon_pmu *pmu, struct perfmon_dev *dev)
{
  assert (dev);
  assert (!pmu->dev);
  pmu->dev = dev;
}

static struct perfmon_dev*
perfmon_pmu_get_dev (const struct perfmon_pmu *pmu)
{
  return (pmu->dev);
}

static void
perfmon_pmu_handle_overflow_intr (const struct perfmon_pmu *pmu)
{
  pmu->dev->ops->handle_overflow_intr ();
}

static int
perfmon_pmu_translate (const struct perfmon_pmu *pmu,
                       unsigned int *raw_event_id,
                       unsigned int event_id)
{
  return (pmu->dev ?
          pmu->dev->ops->translate (raw_event_id, event_id) : ENODEV);
}

static int
perfmon_pmu_alloc_pmc_id (const struct perfmon_pmu *pmu,
                          unsigned int *pmc_idp,
                          unsigned int pmc_index,
                          unsigned int raw_event_id)
{
  if (!pmu->dev)
    return (ENODEV);

  unsigned int pmc_id;
  int error = pmu->dev->ops->alloc (&pmc_id, pmc_index, raw_event_id);

  if (error)
    return (error);

  *pmc_idp = pmc_id;
  return (0);
}

static void
perfmon_pmu_free_pmc_id (const struct perfmon_pmu *pmu, unsigned int pmc_id)
{
  assert (pmu->dev);
  pmu->dev->ops->free (pmc_id);
}

static struct perfmon_pmc *
perfmon_pmu_find_unused_pmc (struct perfmon_pmu *pmu)
{
  for (size_t i = 0; i < ARRAY_SIZE (pmu->pmcs); i++)
    {
      _Auto pmc = perfmon_pmu_get_pmc (pmu, i);
      if (!perfmon_pmc_used (pmc))
        return (pmc);
    }

  return (NULL);
}

static int
perfmon_pmu_alloc_pmc (struct perfmon_pmu *pmu, struct perfmon_pmc **pmcp,
                       unsigned int raw_event_id)
{
  _Auto pmc = perfmon_pmu_find_unused_pmc (pmu);
  if (! pmc)
    return (EAGAIN);

  unsigned int pmc_index = perfmon_pmu_get_pmc_index (pmu, pmc),
               pmc_id = 0;
  int error = perfmon_pmu_alloc_pmc_id (pmu, &pmc_id, pmc_index, raw_event_id);

  if (error)
    return (error);

  perfmon_pmc_use (pmc, pmc_id, raw_event_id);
  *pmcp = pmc;
  return (0);
}

static void
perfmon_pmu_free_pmc (struct perfmon_pmu *pmu, struct perfmon_pmc *pmc)
{
  assert (!perfmon_pmc_used (pmc));
  unsigned int pmc_id = perfmon_pmc_id (pmc);
  perfmon_pmu_free_pmc_id (pmu, pmc_id);
}

static struct perfmon_pmc *
perfmon_pmu_get_pmc_by_raw_event_id (struct perfmon_pmu *pmu,
                                     unsigned int raw_event_id)
{
  for (size_t i = 0; i < ARRAY_SIZE (pmu->pmcs); i++)
    {
      _Auto pmc = perfmon_pmu_get_pmc (pmu, i);
      if (!perfmon_pmc_used (pmc))
        continue;
      else if (perfmon_pmc_raw_event_id (pmc) == raw_event_id)
        return (pmc);
    }

  return (NULL);
}

static int
perfmon_pmu_take_pmc (struct perfmon_pmu *pmu, struct perfmon_pmc **pmcp,
                      unsigned int raw_event_id)
{
  spinlock_lock (&pmu->lock);
  int error;
  _Auto pmc = perfmon_pmu_get_pmc_by_raw_event_id (pmu, raw_event_id);

  if (pmc)
    {
      perfmon_pmc_ref (pmc);
      error = 0;
    }
  else
    {
      error = perfmon_pmu_alloc_pmc (pmu, &pmc, raw_event_id);

      if (error)
        pmc = NULL;
    }

  spinlock_unlock (&pmu->lock);

  if (! error)
    *pmcp = pmc;

  return (error);
}

static void
perfmon_pmu_put_pmc (struct perfmon_pmu *pmu, struct perfmon_pmc *pmc)
{
  spinlock_lock (&pmu->lock);
  perfmon_pmc_unref (pmc);

  if (!perfmon_pmc_used (pmc))
    perfmon_pmu_free_pmc (pmu, pmc);

  spinlock_unlock (&pmu->lock);
}

static int
perfmon_check_event_args (unsigned int id, unsigned int flags)
{
  if (!((flags & PERFMON_EF_PUBLIC_MASK) == flags) ||
      !(flags & PERFMON_EF_RAW) || id < PERFMON_NR_GENERIC_EVENTS ||
      !((flags & (PERFMON_EF_KERN | PERFMON_EF_USER))))
    return (EINVAL);

  return (0);
}

int
perfmon_event_init (struct perfmon_event *event, unsigned int id,
                    unsigned int flags)
{
  int error = perfmon_check_event_args (id, flags);

  if (error)
    return (error);

  spinlock_init (&event->lock);
  event->flags = flags;
  event->id = id;
  event->value = 0;
  return (0);
}

static bool
perfmon_event_type_cpu (const struct perfmon_event *event)
{
  return (event->flags & PERFMON_EF_TYPE_CPU);
}

static void
perfmon_event_set_type_cpu (struct perfmon_event *event)
{
  event->flags |= PERFMON_EF_TYPE_CPU;
}

static void
perfmon_event_clear_type_cpu (struct perfmon_event *event)
{
  event->flags &= ~PERFMON_EF_TYPE_CPU;
}

static bool
perfmon_event_attached (const struct perfmon_event *event)
{
  return (event->flags & PERFMON_EF_ATTACHED);
}

static unsigned int
perfmon_event_pmc_index (const struct perfmon_event *event)
{
  assert (perfmon_event_attached (event));
  return (event->pmc_index);
}

static void __init
perfmon_cpu_pmc_init (struct perfmon_cpu_pmc *cpu_pmc)
{
  cpu_pmc->nr_refs = 0;
}

static bool
perfmon_cpu_pmc_used (const struct perfmon_cpu_pmc *cpu_pmc)
{
  return (cpu_pmc->nr_refs != 0);
}

static void
perfmon_cpu_pmc_use (struct perfmon_cpu_pmc *cpu_pmc, unsigned int pmc_id,
                     unsigned int raw_event_id, uint64_t raw_value)
{
  assert (!perfmon_cpu_pmc_used (cpu_pmc));

  cpu_pmc->nr_refs = 1;
  cpu_pmc->pmc_id = pmc_id;
  cpu_pmc->raw_event_id = raw_event_id;
  cpu_pmc->raw_value = raw_value;
  cpu_pmc->value = 0;
}

static void
perfmon_cpu_pmc_ref (struct perfmon_cpu_pmc *cpu_pmc)
{
  assert (perfmon_cpu_pmc_used (cpu_pmc));
  ++cpu_pmc->nr_refs;
}

static void
perfmon_cpu_pmc_unref (struct perfmon_cpu_pmc *cpu_pmc)
{
  assert (perfmon_cpu_pmc_used (cpu_pmc));
  --cpu_pmc->nr_refs;
}

static unsigned int
perfmon_cpu_pmc_id (const struct perfmon_cpu_pmc *cpu_pmc)
{
  return (cpu_pmc->pmc_id);
}

static bool
perfmon_cpu_pmc_update (struct perfmon_cpu_pmc *cpu_pmc, uint64_t raw_value,
                        unsigned int pmc_width)
{
  uint64_t delta = raw_value - cpu_pmc->raw_value;
  bool overflowed;

  if (pmc_width == 64)
    overflowed = false;
  else if (raw_value >= cpu_pmc->raw_value)
    overflowed = false;
  else
    {
      overflowed = true;
      delta += (uint64_t)1 << pmc_width;
    }

  cpu_pmc->value += delta;
  cpu_pmc->raw_value = raw_value;
  return (overflowed);
}

static uint64_t
perfmon_cpu_pmc_get_value (const struct perfmon_cpu_pmc *cpu_pmc)
{
  return (cpu_pmc->value);
}

static struct perfmon_cpu_pmc*
perfmon_cpu_pmu_get_pmc (struct perfmon_cpu_pmu *cpu_pmu, unsigned int index)
{
  assert (index < ARRAY_SIZE (cpu_pmu->pmcs));
  return (&cpu_pmu->pmcs[index]);
}

static void
perfmon_cpu_pmu_start (struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_id,
                       unsigned int raw_event_id)
{
  cpu_pmu->dev->ops->start (pmc_id, raw_event_id);
}

static void
perfmon_cpu_pmu_stop (struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_id)
{
  cpu_pmu->dev->ops->stop (pmc_id);
}

static uint64_t
perfmon_cpu_pmu_read (const struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_id)
{
  return (cpu_pmu->dev->ops->read (pmc_id));
}

static void
perfmon_cpu_pmu_use_pmc (struct perfmon_cpu_pmu *cpu_pmu,
                         struct perfmon_cpu_pmc *cpu_pmc,
                         unsigned int pmc_id,
                         unsigned int raw_event_id)
{
  perfmon_cpu_pmu_start (cpu_pmu, pmc_id, raw_event_id);
  uint64_t raw_value = perfmon_cpu_pmu_read (cpu_pmu, pmc_id);
  perfmon_cpu_pmc_use (cpu_pmc, pmc_id, raw_event_id, raw_value);
}

static void
perfmon_cpu_pmu_update_pmc (struct perfmon_cpu_pmu *cpu_pmu,
                            struct perfmon_cpu_pmc *cpu_pmc)
{
  uint64_t raw_value = perfmon_cpu_pmu_read (cpu_pmu,
                                             perfmon_cpu_pmc_id (cpu_pmc));
  bool overflowed = perfmon_cpu_pmc_update (cpu_pmc, raw_value,
                                            cpu_pmu->dev->pmc_width);

  if (overflowed)
    syscnt_inc (&cpu_pmu->sc_nr_overflows);
}

static void
perfmon_cpu_pmu_check_overflow (void *arg)
{
  assert (!cpu_intr_enabled ());

  struct perfmon_cpu_pmu *cpu_pmu = arg;
  assert (cpu_pmu->cpu == cpu_id ());

  for (size_t i = 0; i < ARRAY_SIZE (cpu_pmu->pmcs); i++)
    {
      _Auto cpu_pmc = perfmon_cpu_pmu_get_pmc (cpu_pmu, i);

      if (!perfmon_cpu_pmc_used (cpu_pmc))
        continue;

      perfmon_cpu_pmu_update_pmc (cpu_pmu, cpu_pmc);
    }
}

static void
perfmon_cpu_pmu_poll (struct timer *timer)
{
  _Auto cpu_pmu = structof (timer, struct perfmon_cpu_pmu, poll_timer);
  xcall_call (perfmon_cpu_pmu_check_overflow, cpu_pmu, cpu_pmu->cpu);
  timer_schedule (timer, timer_get_time (timer) + cpu_pmu->dev->poll_interval);
}

static void __init
perfmon_cpu_pmu_init (struct perfmon_cpu_pmu *cpu_pmu, unsigned int cpu,
                      struct perfmon_dev *dev)
{
  cpu_pmu->dev = dev;
  cpu_pmu->cpu = cpu;

  for (size_t i = 0; i < ARRAY_SIZE (cpu_pmu->pmcs); i++)
    perfmon_cpu_pmc_init (perfmon_cpu_pmu_get_pmc (cpu_pmu, i));

  if (dev->ops->handle_overflow_intr == NULL)
    {
      assert (dev->poll_interval != 0);

      /*
       * XXX Ideally, this would be an interrupt timer instead of a high
       * priority one, but it can't be because the handler performs
       * cross-calls to remote processors, which requires that interrupts
       * be enabled. This is one potential user of CPU-bound timers.
       */
      timer_init (&cpu_pmu->poll_timer, perfmon_cpu_pmu_poll, TIMER_HIGH_PRIO);
      timer_schedule (&cpu_pmu->poll_timer, dev->poll_interval);
    }

  char name[SYSCNT_NAME_SIZE];
  snprintf (name, sizeof (name), "perfmon_nr_overflows/%u", cpu);
  syscnt_register (&cpu_pmu->sc_nr_overflows, name);
}

static uint64_t
perfmon_cpu_pmu_load (struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_index,
                      unsigned int pmc_id, unsigned int raw_event_id)
{
  assert (!cpu_intr_enabled ());
  _Auto cpu_pmc = perfmon_cpu_pmu_get_pmc (cpu_pmu, pmc_index);

  if (perfmon_cpu_pmc_used (cpu_pmc))
    {
      perfmon_cpu_pmc_ref (cpu_pmc);
      perfmon_cpu_pmu_update_pmc (cpu_pmu, cpu_pmc);
    }
  else
    perfmon_cpu_pmu_use_pmc (cpu_pmu, cpu_pmc, pmc_id, raw_event_id);

  return (perfmon_cpu_pmc_get_value (cpu_pmc));
}

static uint64_t
perfmon_cpu_pmu_unload (struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_index)
{
  assert (!cpu_intr_enabled ());

  _Auto cpu_pmc = perfmon_cpu_pmu_get_pmc (cpu_pmu, pmc_index);
  unsigned int pmc_id = perfmon_cpu_pmc_id (cpu_pmc);

  perfmon_cpu_pmu_update_pmc (cpu_pmu, cpu_pmc);
  uint64_t value = perfmon_cpu_pmc_get_value (cpu_pmc);

  perfmon_cpu_pmc_unref (cpu_pmc);

  if (!perfmon_cpu_pmc_used (cpu_pmc))
    perfmon_cpu_pmu_stop (cpu_pmu, pmc_id);

  return (value);
}

static uint64_t
perfmon_cpu_pmu_sync (struct perfmon_cpu_pmu *cpu_pmu, unsigned int pmc_index)
{
  assert (!cpu_intr_enabled ());

  _Auto cpu_pmc = perfmon_cpu_pmu_get_pmc (cpu_pmu, pmc_index);
  perfmon_cpu_pmu_update_pmc (cpu_pmu, cpu_pmc);
  return (perfmon_cpu_pmc_get_value (cpu_pmc));
}

static void
perfmon_td_pmc_init (struct perfmon_td_pmc *td_pmc)
{
  td_pmc->nr_refs = 0;
  td_pmc->loaded = false;
  td_pmc->value = 0;
}

static bool
perfmon_td_pmc_used (const struct perfmon_td_pmc *td_pmc)
{
  return (td_pmc->nr_refs != 0);
}

static void
perfmon_td_pmc_use (struct perfmon_td_pmc *td_pmc, unsigned int pmc_id,
                    unsigned int raw_event_id)
{
  assert (!perfmon_td_pmc_used (td_pmc));

  td_pmc->nr_refs = 1;
  td_pmc->loaded = false;
  td_pmc->pmc_id = pmc_id;
  td_pmc->raw_event_id = raw_event_id;
  td_pmc->value = 0;
}

static unsigned int
perfmon_td_pmc_id (const struct perfmon_td_pmc *td_pmc)
{
  return (td_pmc->pmc_id);
}

static unsigned int
perfmon_td_pmc_raw_event_id (const struct perfmon_td_pmc *td_pmc)
{
  return (td_pmc->raw_event_id);
}

static void
perfmon_td_pmc_ref (struct perfmon_td_pmc *td_pmc)
{
  assert (perfmon_td_pmc_used (td_pmc));
  ++td_pmc->nr_refs;
}

static void
perfmon_td_pmc_unref (struct perfmon_td_pmc *td_pmc)
{
  assert (perfmon_td_pmc_used (td_pmc));
  --td_pmc->nr_refs;
}

static bool
perfmon_td_pmc_loaded (const struct perfmon_td_pmc *td_pmc)
{
  return (td_pmc->loaded);
}

static void
perfmon_td_pmc_load (struct perfmon_td_pmc *td_pmc, uint64_t cpu_pmc_value)
{
  assert (!perfmon_td_pmc_loaded (td_pmc));

  td_pmc->cpu_pmc_value = cpu_pmc_value;
  td_pmc->loaded = true;
}

static void
perfmon_td_pmc_update (struct perfmon_td_pmc *td_pmc, uint64_t cpu_pmc_value)
{
  assert (perfmon_td_pmc_loaded (td_pmc));
  uint64_t delta = cpu_pmc_value - td_pmc->cpu_pmc_value;
  td_pmc->cpu_pmc_value = cpu_pmc_value;
  td_pmc->value += delta;
}

static void
perfmon_td_pmc_unload (struct perfmon_td_pmc *td_pmc, uint64_t cpu_pmc_value)
{
  perfmon_td_pmc_update (td_pmc, cpu_pmc_value);
  td_pmc->loaded = false;
}

static uint64_t
perfmon_td_pmc_read (const struct perfmon_td_pmc *td_pmc)
{
  return (td_pmc->value);
}

static unsigned int
perfmon_td_get_pmc_index (const struct perfmon_td *td,
                          const struct perfmon_td_pmc *td_pmc)
{
  size_t pmc_index = td_pmc - td->pmcs;
  assert (pmc_index < ARRAY_SIZE (td->pmcs));
  return (pmc_index);
}

static struct perfmon_td_pmc*
perfmon_td_get_pmc (struct perfmon_td *td, unsigned int index)
{
  assert (index < ARRAY_SIZE (td->pmcs));
  return (&td->pmcs[index]);
}

void
perfmon_td_init (struct perfmon_td *td)
{
  spinlock_init (&td->lock);

  for (size_t i = 0; i < ARRAY_SIZE (td->pmcs); i++)
    perfmon_td_pmc_init (perfmon_td_get_pmc (td, i));
}

static void
perfmon_td_load_pmc (struct perfmon_td *td, struct perfmon_td_pmc *td_pmc)
{
  _Auto cpu_pmu = perfmon_get_local_cpu_pmu ();
  unsigned int pmc_index = perfmon_td_get_pmc_index (td, td_pmc),
               pmc_id = perfmon_td_pmc_id (td_pmc),
               raw_event_id = perfmon_td_pmc_raw_event_id (td_pmc);
  uint64_t cpu_pmc_value = perfmon_cpu_pmu_load (cpu_pmu, pmc_index,
                                                 pmc_id, raw_event_id);
  perfmon_td_pmc_load (td_pmc, cpu_pmc_value);
}

static void
perfmon_td_unload_pmc (struct perfmon_td *td, struct perfmon_td_pmc *td_pmc)
{
  _Auto cpu_pmu = perfmon_get_local_cpu_pmu ();
  unsigned int pmc_index = perfmon_td_get_pmc_index (td, td_pmc);
  uint64_t cpu_pmc_value = perfmon_cpu_pmu_unload (cpu_pmu, pmc_index);
  perfmon_td_pmc_unload (td_pmc, cpu_pmc_value);
}

static void
perfmon_td_update_pmc (struct perfmon_td *td, struct perfmon_td_pmc *td_pmc)
{
  _Auto cpu_pmu = perfmon_get_local_cpu_pmu ();
  unsigned int pmc_index = perfmon_td_get_pmc_index (td, td_pmc);
  uint64_t cpu_pmc_value = perfmon_cpu_pmu_sync (cpu_pmu, pmc_index);
  perfmon_td_pmc_update (td_pmc, cpu_pmc_value);
}

void
perfmon_td_load (struct perfmon_td *td)
{
  unsigned int pmc_index, pmc_id, raw_event_id;
  struct perfmon_td_pmc *td_pmc;
  uint64_t cpu_pmc_value;

  assert (!cpu_intr_enabled ());
  assert (!thread_preempt_enabled ());

  _Auto cpu_pmu = perfmon_get_local_cpu_pmu ();

  spinlock_lock (&td->lock);
  for (size_t i = 0; i < ARRAY_SIZE (td->pmcs); i++)
    {
      _Auto td_pmc = perfmon_td_get_pmc (td, i);

      if (!perfmon_td_pmc_used (td_pmc) || perfmon_td_pmc_loaded (td_pmc))
        continue;

      unsigned int pmc_index = perfmon_td_get_pmc_index (td, td_pmc),
                   pmc_id = perfmon_td_pmc_id (td_pmc),
                   raw_event_id = perfmon_td_pmc_raw_event_id (td_pmc);
      cpu_pmc_value = perfmon_cpu_pmu_load (cpu_pmu, pmc_index,
                                            pmc_id, raw_event_id);
      perfmon_td_pmc_load (td_pmc, cpu_pmc_value);
    }

  spinlock_unlock (&td->lock);
}

void
perfmon_td_unload (struct perfmon_td *td)
{
  assert (!cpu_intr_enabled ());
  assert (!thread_preempt_enabled ());

  _Auto cpu_pmu = perfmon_get_local_cpu_pmu ();
  spinlock_lock (&td->lock);

  for (size_t i = 0; i < ARRAY_SIZE (td->pmcs); i++)
    {
      _Auto td_pmc = perfmon_td_get_pmc (td, i);

      if (!perfmon_td_pmc_loaded (td_pmc))
        continue;

      unsigned int pmc_index = perfmon_td_get_pmc_index (td, td_pmc);
      uint64_t cpu_pmc_value = perfmon_cpu_pmu_unload (cpu_pmu, pmc_index);
      perfmon_td_pmc_unload (td_pmc, cpu_pmc_value);
    }

  spinlock_unlock (&td->lock);
}

static void
perfmon_event_load (struct perfmon_event *event, uint64_t pmc_value)
{
  event->pmc_value = pmc_value;
}

static void
perfmon_event_update (struct perfmon_event *event, uint64_t pmc_value)
{
  uint64_t delta = pmc_value - event->pmc_value;
  event->value += delta;
  event->pmc_value = pmc_value;
}

static void
perfmon_event_load_cpu_remote (void *arg)
{
  uint64_t cpu_pmc_value;
  struct perfmon_event *event = arg;
  _Auto cpu_pmu = perfmon_get_local_cpu_pmu ();
  _Auto pmu = perfmon_get_pmu();
  unsigned int pmc_index = perfmon_event_pmc_index (event);
  _Auto pmc = perfmon_pmu_get_pmc (pmu, pmc_index);
  cpu_pmc_value = perfmon_cpu_pmu_load (cpu_pmu, pmc_index,
                                        perfmon_pmc_id (pmc),
                                        perfmon_pmc_raw_event_id (pmc));
  perfmon_event_load (event, cpu_pmc_value);
}

static void
perfmon_event_load_cpu (struct perfmon_event *event, unsigned int cpu)
{
  perfmon_event_set_type_cpu (event);
  event->cpu = cpu;
  xcall_call (perfmon_event_load_cpu_remote, event, cpu);
}

static void
perfmon_event_load_thread_remote (void *arg)
{
  struct perfmon_event *event = arg;
  unsigned int pmc_index = perfmon_event_pmc_index (event);
  _Auto td = thread_get_perfmon_td (event->thread);
  _Auto td_pmc = perfmon_td_get_pmc (td, pmc_index);

  spinlock_lock (&td->lock);

  if (thread_self () != event->thread)
    ;
  else if (perfmon_td_pmc_loaded (td_pmc))
    perfmon_td_update_pmc (td, td_pmc);
  else
    perfmon_td_load_pmc (td, td_pmc);

  uint64_t td_pmc_value = perfmon_td_pmc_read (td_pmc);
  spinlock_unlock (&td->lock);
  perfmon_event_load (event, td_pmc_value);
}

static void
perfmon_event_load_thread (struct perfmon_event *event, struct thread *thread)
{
  _Auto pmu = perfmon_get_pmu ();
  thread_ref (thread);
  event->thread = thread;

  unsigned int pmc_index = perfmon_event_pmc_index (event);
  _Auto pmc = perfmon_pmu_get_pmc (pmu, pmc_index);
  _Auto td = thread_get_perfmon_td (thread);
  _Auto td_pmc = perfmon_td_get_pmc (td, pmc_index);

  cpu_flags_t flags;
  spinlock_lock_intr_save (&td->lock, &flags);

  if (perfmon_td_pmc_used (td_pmc))
    perfmon_td_pmc_ref (td_pmc);
  else
    perfmon_td_pmc_use (td_pmc, perfmon_pmc_id (pmc),
                        perfmon_pmc_raw_event_id (pmc));

  spinlock_unlock_intr_restore (&td->lock, flags);
  xcall_call (perfmon_event_load_thread_remote, event, thread_cpu (thread));
}

static void
perfmon_event_unload_cpu_remote (void *arg)
{
  struct perfmon_event *event = arg;
  _Auto cpu_pmu = perfmon_get_local_cpu_pmu ();
  unsigned int pmc_index = perfmon_event_pmc_index (event);
  uint64_t cpu_pmc_value = perfmon_cpu_pmu_unload (cpu_pmu, pmc_index);
  perfmon_event_update (event, cpu_pmc_value);
}

static void
perfmon_event_unload_cpu (struct perfmon_event *event)
{
  xcall_call (perfmon_event_unload_cpu_remote, event, event->cpu);
  perfmon_event_clear_type_cpu (event);
}

static void
perfmon_event_unload_thread_remote (void *arg)
{
  struct perfmon_event *event = arg;
  unsigned int pmc_index = perfmon_event_pmc_index (event);
  _Auto td = thread_get_perfmon_td (event->thread);
  _Auto td_pmc = perfmon_td_get_pmc (td, pmc_index);

  spinlock_lock (&td->lock);

  if (dthread_self () == event->thread && perfmon_td_pmc_loaded (td_pmc))
    {
      if (perfmon_td_pmc_used (td_pmc))
        perfmon_td_update_pmc (td, td_pmc);
      else
        perfmon_td_unload_pmc (td, td_pmc);
    }

  uint64_t td_pmc_value = perfmon_td_pmc_read (td_pmc);
  spinlock_unlock (&td->lock);
  perfmon_event_update (event, td_pmc_value);
}

static void
perfmon_event_unload_thread (struct perfmon_event *event)
{
  unsigned int pmc_index = perfmon_event_pmc_index (event);
  _Auto td = thread_get_perfmon_td (event->thread);
  _Auto td_pmc = perfmon_td_get_pmc (td, pmc_index);

  cpu_flags_t flags;
  spinlock_lock_intr_save (&td->lock, &flags);
  perfmon_td_pmc_unref (td_pmc);
  spinlock_unlock_intr_restore (&td->lock, flags);

  xcall_call (perfmon_event_unload_thread_remote, event,
              thread_cpu (event->thread));

  thread_unref (event->thread);
  event->thread = NULL;
}

static void
perfmon_event_sync_cpu_remote (void *arg)
{
  struct perfmon_event *event = arg;
  _Auto cpu_pmu = perfmon_get_local_cpu_pmu();
  unsigned int pmc_index = perfmon_event_pmc_index (event);
  uint64_t cpu_pmc_value = perfmon_cpu_pmu_sync (cpu_pmu, pmc_index);
  perfmon_event_update (event, cpu_pmc_value);
}

static void
perfmon_event_sync_cpu (struct perfmon_event *event)
{
  xcall_call (perfmon_event_sync_cpu_remote, event, event->cpu);
}

static void
perfmon_event_sync_thread_remote (void *arg)
{
  struct perfmon_event *event = arg;
  unsigned int pmc_index = perfmon_event_pmc_index (event);
  _Auto td = thread_get_perfmon_td (event->thread);
  _Auto td_pmc = perfmon_td_get_pmc (td, pmc_index);

  spinlock_lock (&td->lock);

  if (thread_self () == event->thread)
    perfmon_td_update_pmc (td, td_pmc);

  uint64_t td_pmc_value = perfmon_td_pmc_read (td_pmc);
  spinlock_unlock (&td->lock);
  perfmon_event_update (event, td_pmc_value);
}

static void
perfmon_event_sync_thread (struct perfmon_event *event)
{
  xcall_call (perfmon_event_sync_thread_remote, event,
              thread_cpu (event->thread));
}

static int
perfmon_event_attach_pmu (struct perfmon_event *event)
{
  unsigned int raw_event_id = 0;
  _Auto pmu = perfmon_get_pmu ();

  if (! (event->flags & PERFMON_EF_RAW))
    {
      int error = perfmon_pmu_translate (pmu, &raw_event_id, event->id);

      if (error)
        return (error);
    }

  struct perfmon_pmc *pmc;
  int error = perfmon_pmu_take_pmc (pmu, &pmc, raw_event_id);

  if (error)
    return (error);

  event->pmc_index = perfmon_pmu_get_pmc_index (pmu, pmc);
  event->flags |= PERFMON_EF_ATTACHED;
  event->value = 0;
  return (0);
}

static void
perfmon_event_detach_pmu (struct perfmon_event *event)
{
  _Auto pmu = perfmon_get_pmu();
  _Auto pmc = perfmon_pmu_get_pmc (pmu, perfmon_event_pmc_index (event));
  perfmon_pmu_put_pmc (pmu, pmc);
  event->flags &= ~PERFMON_EF_ATTACHED;
}

int
perfmon_event_attach (struct perfmon_event *event, struct thread *thread)
{
  int error;

  spinlock_lock (&event->lock);
  if (perfmon_event_attached (event))
    {
      error = EINVAL;
      goto error;
    }

  error = perfmon_event_attach_pmu (event);

  if (! error)
    perfmon_event_load_thread (event, thread);

error:
  spinlock_unlock (&event->lock);
  return (error);
}

int
perfmon_event_attach_cpu (struct perfmon_event *event, unsigned int cpu)
{
  int error;

  if (cpu >= cpu_count ())
    return (EINVAL);

  spinlock_lock (&event->lock);

  if (perfmon_event_attached (event))
    {
      error = EINVAL;
      goto out;
    }

  error = perfmon_event_attach_pmu (event);
  if (! error)
    perfmon_event_load_cpu (event, cpu);

out:
  spinlock_unlock (&event->lock);
  return (error);
}

int
perfmon_event_detach (struct perfmon_event *event)
{
  int error;

  spinlock_lock (&event->lock);

  if (!perfmon_event_attached (event))
    {
      error = EINVAL;
      goto out;
    }

  if (perfmon_event_type_cpu (event))
    perfmon_event_unload_cpu (event);
  else
    perfmon_event_unload_thread (event);

  perfmon_event_detach_pmu (event);
  error = 0;

out:
  spinlock_unlock (&event->lock);
  return (error);
}

uint64_t
perfmon_event_read (struct perfmon_event *event)
{
  spinlock_lock (&event->lock);
  if (perfmon_event_attached (event))
    {
      if (perfmon_event_type_cpu (event))
        perfmon_event_sync_cpu (event);
      else
        perfmon_event_sync_thread (event);
    }

  uint64_t value = event->value;
  spinlock_unlock (&event->lock);
  return (value);
}

static uint64_t __init
perfmon_compute_poll_interval (uint64_t pmc_width)
{
  uint64_t cycles;

  if (pmc_width == 64)
    cycles = (uint64_t)-1;
  else
    cycles = (uint64_t)1 << pmc_width;

  /*
   * Assume an unrealistically high upper bound on the number of
   * events per cycle to otbain a comfortable margin of safety.
   */
  cycles /= 100;
  uint64_t time = cycles / (cpu_get_freq () / 1000);

  if (time < PERFMON_MIN_POLL_INTERVAL)
    {
      log_warning ("perfmon: invalid poll interval %llu, forced to %llu",
                   time, PERFMON_MIN_POLL_INTERVAL);
      time = PERFMON_MIN_POLL_INTERVAL;
    }

  return (clock_ticks_from_ms (time));
}

void __init
perfmon_register (struct perfmon_dev *dev)
{
  const struct perfmon_dev_ops *ops = dev->ops;
  assert (ops->translate && ops->alloc && ops->free
          && ops->start && ops->stop && ops->read);
  assert (dev->pmc_width <= 64);

  if (!dev->ops->handle_overflow_intr && !dev->poll_interval)
    dev->poll_interval = perfmon_compute_poll_interval (dev->pmc_width);

  perfmon_pmu_set_dev (perfmon_get_pmu (), dev);
}

void
perfmon_overflow_intr (void)
{
  perfmon_pmu_handle_overflow_intr (perfmon_get_pmu ());
}

void
perfmon_report_overflow (unsigned int pmc_index)
{
  assert (!cpu_intr_enabled ());
  assert (!thread_preempt_enabled ());

  _Auto cpu_pmu = perfmon_get_local_cpu_pmu();
  _Auto cpu_pmc = perfmon_cpu_pmu_get_pmc (cpu_pmu, pmc_index);
  perfmon_cpu_pmu_update_pmc (cpu_pmu, cpu_pmc);
}

static int __init
perfmon_bootstrap (void)
{
  perfmon_pmu_init (perfmon_get_pmu ());
  return (0);
}

INIT_OP_DEFINE (perfmon_bootstrap,
                INIT_OP_DEP (log_setup, true),
                INIT_OP_DEP (spinlock_setup, true));

static int __init
perfmon_setup (void)
{
  struct perfmon_dev *dev = perfmon_pmu_get_dev (perfmon_get_pmu ());

  if (!dev)
    return (ENODEV);

  for (uint32_t cpu = 0; cpu < cpu_count (); ++cpu)
    perfmon_cpu_pmu_init (perfmon_get_cpu_pmu (cpu), cpu, dev);

  return (0);
}

INIT_OP_DEFINE (perfmon_setup,
                INIT_OP_DEP (boot_setup_pmu, true),
                INIT_OP_DEP (cpu_mp_probe, true),
                INIT_OP_DEP (cpu_setup, true),
                INIT_OP_DEP (percpu_setup, true),
                INIT_OP_DEP (perfmon_bootstrap, true),
                INIT_OP_DEP (spinlock_setup, true),
                INIT_OP_DEP (syscnt_setup, true));