/* -*- Mode: C++; tab-width: 8; c-basic-offset: 2; indent-tabs-mode: nil; -*- */
// This file is included from PerfCounters.cc

static bool has_kvm_in_txcp_bug;
static bool has_xen_pmi_bug;
static bool supports_txcp;

/**
 * Return the detected, known microarchitecture of this CPU, or don't
 * return; i.e. never return UnknownCpu.
 */
static CpuMicroarch compute_cpu_microarch() {
  auto cpuid_vendor = cpuid(CPUID_GETVENDORSTRING, 0);
  char vendor[12];
  memcpy(&vendor[0], &cpuid_vendor.ebx, 4);
  memcpy(&vendor[4], &cpuid_vendor.edx, 4);
  memcpy(&vendor[8], &cpuid_vendor.ecx, 4);
  if (strncmp(vendor, "GenuineIntel", sizeof(vendor)) &&
      strncmp(vendor, "AuthenticAMD", sizeof(vendor))) {
    CLEAN_FATAL() << "Unknown CPU vendor '" << vendor << "'";
  }

  auto cpuid_data = cpuid(CPUID_GETFEATURES, 0);
  unsigned int cpu_type = cpuid_data.eax & 0xF0FF0;
  unsigned int ext_family = (cpuid_data.eax >> 20) & 0xff;
  switch (cpu_type) {
    case 0x006F0:
    case 0x10660:
      return IntelMerom;
    case 0x10670:
    case 0x106D0:
      return IntelPenryn;
    case 0x106A0:
    case 0x106E0:
    case 0x206E0:
      return IntelNehalem;
    case 0x20650:
    case 0x206C0:
    case 0x206F0:
      return IntelWestmere;
    case 0x206A0:
    case 0x206D0:
    case 0x306e0:
      return IntelSandyBridge;
    case 0x306A0:
      return IntelIvyBridge;
    case 0x306C0: /* Devil's Canyon */
    case 0x306F0:
    case 0x40650:
    case 0x40660:
      return IntelHaswell;
    case 0x306D0:
    case 0x40670:
    case 0x406F0:
    case 0x50660:
      return IntelBroadwell;
    case 0x406e0:
    case 0x50650:
    case 0x506e0:
      return IntelSkylake;
    case 0x30670:
    case 0x406c0:
    case 0x50670:
      return IntelSilvermont;
    case 0x506f0:
    case 0x706a0:
    case 0x506c0:
      return IntelGoldmont;
    case 0x706e0:
    case 0x606a0:
      return IntelIcelake;
    case 0x806c0:
    case 0x806d0:
      return IntelTigerlake;
    case 0x806e0:
    case 0x906e0:
      return IntelKabylake;
    case 0xa0650:
    case 0xa0660:
      return IntelCometlake;
    case 0xa0670:
      return IntelRocketlake;
    case 0x90670:
    case 0x906a0:
      return IntelAlderlake;
    case 0xb0670:
      return IntelRaptorlake;
    case 0x806f0:
      return IntelSapphireRapid;
    case 0x30f00:
      return AMDF15R30;
    case 0x00f10: // Naples, Whitehaven, Summit Ridge, Snowy Owl (Zen), Milan (Zen 3) (UNTESTED)
    case 0x10f10: // Raven Ridge, Great Horned Owl (Zen) (UNTESTED)
    case 0x10f80: // Banded Kestrel (Zen), Picasso (Zen+) (UNTESTED)
    case 0x20f00: // Dali (Zen) (UNTESTED)
    case 0x00f80: // Colfax, Pinnacle Ridge (Zen+) (UNTESTED)
    case 0x30f10: // Rome, Castle Peak (Zen 2)
    case 0x60f00: // Renoir (Zen 2) (UNTESTED)
    case 0x70f10: // Matisse (Zen 2) (UNTESTED)
    case 0x60f80: // Lucienne
    case 0x90f00: // Van Gogh (Zen 2)
      if (ext_family == 8 || ext_family == 0xa) {
        return AMDZen;
      } else if (ext_family == 3) {
        return AMDF15R30;
      }
      break;
    case 0x20f10: // Vermeer (Zen 3)
    case 0x50f00: // Cezanne (Zen 3)
    case 0x40f40: // Rembrandt (Zen 3+)
    case 0x60f10: // Raphael (Zen 4)
      if (ext_family == 0xa) {
        return AMDZen;
      }
    default:
      break;
  }

  if (!strncmp(vendor, "AuthenticAMD", sizeof(vendor))) {
    CLEAN_FATAL() << "AMD CPU type " << HEX(cpu_type) <<
                     " (ext family " << HEX(ext_family) << ") unknown";
  } else {
    CLEAN_FATAL() << "Intel CPU type " << HEX(cpu_type) << " unknown";
  }
  return UnknownCpu; // not reached
}

static std::vector<CpuMicroarch> compute_cpu_microarchs() {
  return { compute_cpu_microarch() };
}

static void check_for_kvm_in_txcp_bug(const perf_event_attrs &perf_attr) {
  int64_t count = 0;
  struct perf_event_attr attr = perf_attr.ticks;
  attr.config |= IN_TXCP;
  attr.sample_period = 0;
  bool disabled_txcp;
  ScopedFd fd = start_counter(0, -1, &attr, &disabled_txcp);
  if (fd.is_open() && !disabled_txcp) {
    ioctl(fd, PERF_EVENT_IOC_DISABLE, 0);
    ioctl(fd, PERF_EVENT_IOC_ENABLE, 0);
    do_branches();
    count = read_counter(fd);
  }

  supports_txcp = count > 0;
  has_kvm_in_txcp_bug = supports_txcp && count < NUM_BRANCHES;
  LOG(debug) << "supports txcp=" << supports_txcp;
  LOG(debug) << "has_kvm_in_txcp_bug=" << has_kvm_in_txcp_bug
             << " count=" << count;
}

static void check_for_xen_pmi_bug(const perf_event_attrs &perf_attr) {
  int32_t count = -1;
  struct perf_event_attr attr = perf_attr.ticks;
  attr.sample_period = NUM_BRANCHES - 1;
  ScopedFd fd = start_counter(0, -1, &attr);
  if (fd.is_open()) {
    // Do NUM_BRANCHES conditional branches that can't be optimized out.
    // 'accumulator' is always odd and can't be zero
    uint32_t accumulator = uint32_t(rand()) * 2 + 1;
    int raw_fd = fd;
    asm volatile(
#if defined(__x86_64__)
        "mov %[_SYS_ioctl], %%rax;"
        "mov %[raw_fd], %%edi;"
        "xor %%rdx, %%rdx;"
        "mov %[_PERF_EVENT_IOC_ENABLE], %%rsi;"
        "syscall;"
        "cmp $-4095, %%rax;"
        "jae 2f;"
        "mov %[_SYS_ioctl], %%rax;"
        "mov %[_PERF_EVENT_IOC_RESET], %%rsi;"
        "syscall;"
        // From this point on all conditional branches count!
        "cmp $-4095, %%rax;"
        "jae 2f;"
        // Reset the counter period to the desired value.
        "mov %[_SYS_ioctl], %%rax;"
        "mov %[_PERF_EVENT_IOC_PERIOD], %%rsi;"
        "mov %[period], %%rdx;"
        "syscall;"
        "cmp $-4095, %%rax;"
        "jae 2f;"
        "mov %[_iterations], %%rax;"
        "1: dec %%rax;"
        // Multiply by 7.
        "mov %[accumulator], %%edx;"
        "shl $3, %[accumulator];"
        "sub %%edx, %[accumulator];"
        // Add 2.
        "add $2, %[accumulator];"
        // Mask off bits.
        "and $0xffffff, %[accumulator];"
        // And loop.
        "test %%rax, %%rax;"
        "jnz 1b;"
        "mov %[_PERF_EVENT_IOC_DISABLE], %%rsi;"
        "mov %[_SYS_ioctl], %%rax;"
        "xor %%rdx, %%rdx;"
        // We didn't touch rdi.
        "syscall;"
        "cmp $-4095, %%rax;"
        "jae 2f;"
        "movl $0, %[count];"
        "2: nop;"
#elif defined(__i386__)
        "mov %[_SYS_ioctl], %%eax;"
        "mov %[raw_fd], %%ebx;"
        "xor %%edx, %%edx;"
        "mov %[_PERF_EVENT_IOC_ENABLE], %%ecx;"
        "int $0x80;"
        "cmp $-4095, %%eax;"
        "jae 2f;"
        "mov %[_SYS_ioctl], %%eax;"
        "mov %[_PERF_EVENT_IOC_RESET], %%ecx;"
        "int $0x80;"
        // From this point on all conditional branches count!
        "cmp $-4095, %%eax;"
        "jae 2f;"
        // Reset the counter period to the desired value.
        "mov %[_SYS_ioctl], %%eax;"
        "mov %[_PERF_EVENT_IOC_PERIOD], %%ecx;"
        "mov %[period], %%edx;"
        "int $0x80;"
        "cmp $-4095, %%eax;"
        "jae 2f;"
        "mov %[_iterations], %%eax;"
        "1: dec %%eax;"
        // Multiply by 7.
        "mov %[accumulator], %%edx;"
        "shll $3, %[accumulator];"
        "sub %%edx, %[accumulator];"
        // Add 2.
        "addl $2, %[accumulator];"
        // Mask off bits.
        "andl $0xffffff, %[accumulator];"
        // And loop.
        "test %%eax, %%eax;"
        "jnz 1b;"
        "mov %[_PERF_EVENT_IOC_DISABLE], %%ecx;"
        "mov %[_SYS_ioctl], %%eax;"
        "xor %%edx, %%edx;"
        // We didn't touch rdi.
        "int $0x80;"
        "cmp $-4095, %%eax;"
        "jae 2f;"
        "movl $0, %[count];"
        "2: nop;"
#else
#error unknown CPU architecture
#endif
        : [accumulator] "+rm"(accumulator), [count] "=rm"(count)
        : [_SYS_ioctl] "i"(SYS_ioctl),
          [_PERF_EVENT_IOC_DISABLE] "i"(PERF_EVENT_IOC_DISABLE),
          [_PERF_EVENT_IOC_ENABLE] "i"(PERF_EVENT_IOC_ENABLE),
          [_PERF_EVENT_IOC_PERIOD] "i"(PERF_EVENT_IOC_PERIOD),
          [_PERF_EVENT_IOC_RESET] "i"(PERF_EVENT_IOC_RESET),
          // The check for the failure of some of our ioctls is in
          // the measured region, so account for that when looping.
          [_iterations] "i"(NUM_BRANCHES - 2),
          [period] "rm"(&attr.sample_period), [raw_fd] "rm"(raw_fd)
        :
#if defined(__x86_64__)
        "rax", "rdx", "rdi", "rsi"
        // `syscall` clobbers rcx and r11.
        ,
        "rcx", "r11"
#elif defined(__i386__)
        "eax", "ebx", "ecx", "edx"
#else
#error unknown CPU architecture
#endif
        );
    // If things worked above, `count` should have been set to 0.
    if (count == 0) {
      count = read_counter(fd);
    }
    // Use 'accumulator' so it can't be optimized out.
    accumulator_sink = accumulator;
  }

  has_xen_pmi_bug = count > NUM_BRANCHES || count == -1;
  if (has_xen_pmi_bug) {
    LOG(debug) << "has_xen_pmi_bug=" << has_xen_pmi_bug << " count=" << count;
    if (!Flags::get().force_things) {
      FATAL()
          << "Overcount triggered by PMU interrupts detected due to Xen PMU "
             "virtualization bug.\n"
             "Aborting. Retry with -F to override, but it will probably\n"
             "fail.";
    }
  }
}

static void check_for_zen_speclockmap() {
  // When the SpecLockMap optimization is not disabled, rr will not work
  // reliably (e.g. it would work fine on a single process with a single
  // thread, but not more). When the optimization is disabled, the
  // perf counter for retired lock instructions of type SpecLockMapCommit
  // (on PMC 0x25) stays at 0.
  // See more details at https://github.com/rr-debugger/rr/issues/2034.
  struct perf_event_attr attr;
  // 0x25 == RETIRED_LOCK_INSTRUCTIONS - Counts the number of retired locked instructions
  // + 0x08 == SPECLOCKMAPCOMMIT
  init_perf_event_attr(&attr, PERF_TYPE_RAW, 0x510825);

  ScopedFd fd = start_counter(0, -1, &attr);
  if (fd.is_open()) {
    int atomic = 0;
    int64_t count = read_counter(fd);
    // A lock add is known to increase the perf counter we're looking at.
    asm volatile("lock addl $1, %0": "+m" (atomic));
    if (read_counter(fd) == count) {
      LOG(debug) << "SpecLockMap is disabled";
    } else {
      LOG(debug) << "SpecLockMap is not disabled";
      fprintf(stderr,
              "On Zen CPUs, rr will not work reliably unless you disable the "
              "hardware SpecLockMap optimization.\nFor instructions on how to "
              "do this, see https://github.com/rr-debugger/rr/wiki/Zen\n");
    }
  }
}

static void check_for_freeze_on_smi() {
  ScopedFd fd = ScopedFd("/sys/devices/cpu/freeze_on_smi", O_RDONLY);
  if (!fd.is_open()) {
    LOG(debug) << "/sys/devices/cpu/freeze_on_smi not present";
    return;
  }

  char freeze_on_smi = 0;
  ssize_t ret = read(fd, &freeze_on_smi, 1);
  if (ret != 1) {
    FATAL() << "Can't read freeze_on_smi";
  }
  if (freeze_on_smi == 0) {
    LOG(warn) << "Failed to read freeze_on_smi";
  } else if (freeze_on_smi == '1') {
    LOG(debug) << "freeze_on_smi is set";
  } else if (freeze_on_smi == '0') {
    LOG(warn) << "freeze_on_smi is not set";
    if (!Flags::get().suppress_environment_warnings) {
      fprintf(stderr,
              "Freezing performance counters on SMIs should be enabled for maximum rr\n"
              "reliability on Comet Lake and later CPUs. To manually enable this setting, run\n"
              "\techo 1 | sudo tee /sys/devices/cpu/freeze_on_smi\n"
              "On systemd systems, consider putting\n"
              "'w /sys/devices/cpu/freeze_on_smi - - - - 1' into /etc/tmpfiles.d/10-rr.conf\n"
              "to automatically apply this setting on every reboot.\n"
              "See 'man 5 sysfs', 'man 5 tmpfiles.d'.\n"
              "If you are seeing this message, the setting has not been enabled.\n");
    }
  } else {
    LOG(warn) << "Unrecognized freeze_on_smi value " << freeze_on_smi;
  }
}

static void check_for_arch_bugs(perf_event_attrs &perf_attr) {
  DEBUG_ASSERT(rr::perf_attrs.size() == 1);
  CpuMicroarch uarch = (CpuMicroarch)perf_attr.bug_flags;
  if (uarch >= FirstIntel && uarch <= LastIntel) {
    check_for_kvm_in_txcp_bug(perf_attr);
    check_for_xen_pmi_bug(perf_attr);
  }
  if (uarch >= IntelCometlake && uarch <= LastIntel) {
    check_for_freeze_on_smi();
  }
  if (uarch == AMDZen) {
    check_for_zen_speclockmap();
  }
}

static void post_init_pmu_uarchs(std::vector<PmuConfig> &pmu_uarchs)
{
  if (pmu_uarchs.size() != 1) {
    CLEAN_FATAL() << "rr only support a single PMU on x86, "
                  << pmu_uarchs.size() << " specified.";
  }
}

static bool always_recreate_counters(const perf_event_attrs &perf_attr) {
  // When we have the KVM IN_TXCP bug, reenabling the TXCP counter after
  // disabling it does not work.
  DEBUG_ASSERT(perf_attr.checked);
  return perf_attr.has_ioc_period_bug || has_kvm_in_txcp_bug;
}

static void arch_check_restricted_counter() {
  if ((cpuid(CPUID_GETEXTENDEDFEATURES, 0).ebx & HLE_FEATURE_FLAG) &&
    !Flags::get().suppress_environment_warnings) {
    fprintf(stderr,
            "Your CPU supports Hardware Lock Elision but you only have one\n"
            "hardware performance counter available. Record and replay\n"
            "of code that uses HLE will fail unless you alter your\n"
            "configuration to make more than one hardware performance counter\n"
            "available.\n");
  }
}

template <typename Arch>
void PerfCounters::reset_arch_extras() {
  DEBUG_ASSERT(rr::perf_attrs.size() == 1);
  if (supports_txcp) {
    struct perf_event_attr attr = rr::perf_attrs[0].ticks;
    if (has_kvm_in_txcp_bug) {
      // IN_TXCP isn't going to work reliably. Assume that HLE/RTM are not
      // used,
      // and check that.
      attr.sample_period = 0;
      attr.config |= IN_TX;
      fd_ticks_in_transaction = start_counter(tid, fd_ticks_interrupt, &attr);
    } else {
      // Set up a separate counter for measuring ticks, which does not have
      // a sample period and does not count events during aborted
      // transactions.
      // We have to use two separate counters here because the kernel does
      // not support setting a sample_period with IN_TXCP, apparently for
      // reasons related to this Intel note on IA32_PERFEVTSEL2:
      // ``When IN_TXCP=1 & IN_TX=1 and in sampling, spurious PMI may
      // occur and transactions may continuously abort near overflow
      // conditions. Software should favor using IN_TXCP for counting over
      // sampling. If sampling, software should use large “sample-after“
      // value after clearing the counter configured to use IN_TXCP and
      // also always reset the counter even when no overflow condition
      // was reported.''
      attr.sample_period = 0;
      attr.config |= IN_TXCP;
      fd_ticks_measure = start_counter(tid, fd_ticks_interrupt, &attr);
    }
  }
}