There's enough of these floating around randomly to justify using a
structure for keeping them in one place.
Signed-off-by: Zachary Amsden <zamsden@xxxxxxxxxx>
---
arch/x86/kvm/x86.c | 110 ++++++++++++++++++++++++++++++----------------------
1 files changed, 64 insertions(+), 46 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 8bd362b..7bc20ac 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -738,19 +738,24 @@ static void kvm_set_time_scale(uint32_t tsc_khz, struct pvclock_vcpu_time_info *
}
DEFINE_SPINLOCK(kvm_tsc_lock);
-static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz);
-static DEFINE_PER_CPU(unsigned long, cpu_tsc_multiplier);
-static DEFINE_PER_CPU(int, cpu_tsc_shift);
-static DEFINE_PER_CPU(s64, cpu_tsc_offset);
-static DEFINE_PER_CPU(u64, cpu_tsc_measure_base);
-static DEFINE_PER_CPU(atomic_t, cpu_tsc_synchronized);
-static DEFINE_PER_CPU(int, cpu_tsc_generation);
+struct cpu_tsc_vars
+{
+ unsigned long tsc_khz;
+ unsigned long tsc_multiplier;
+ unsigned int tsc_shift;
+ int tsc_generation;
+ s64 tsc_offset;
+ u64 tsc_measure_base;
+ atomic_t tsc_synchronized;
+};
+static DEFINE_PER_CPU(struct cpu_tsc_vars, cpu_tsc_vars);
+
static int tsc_base_cpu = -1;
static unsigned long ref_tsc_khz;
static inline int cpu_is_tsc_synchronized(int cpu)
{
- return (atomic_read(&per_cpu(cpu_tsc_synchronized, cpu)) != 0);
+ return (atomic_read(&per_cpu(cpu_tsc_vars, cpu).tsc_synchronized) != 0);
}
static inline unsigned long div_precise(unsigned long hi, unsigned long lo,
@@ -808,12 +813,12 @@ static inline unsigned long mult_precise(unsigned long val, unsigned long mult,
return bot;
}
-static inline u64 compute_ref_tsc(int cpu)
+static inline u64 compute_ref_tsc(void)
{
- u64 tsc = native_read_tsc() - per_cpu(cpu_tsc_measure_base, cpu);
- tsc = mult_precise(tsc, per_cpu(cpu_tsc_multiplier, cpu),
- per_cpu(cpu_tsc_shift, cpu));
- return tsc + per_cpu(cpu_tsc_offset, cpu);
+ struct cpu_tsc_vars *cv = &__get_cpu_var(cpu_tsc_vars);
+ u64 tsc = native_read_tsc() - cv->tsc_measure_base;
+ tsc = mult_precise(tsc, cv->tsc_multiplier, cv->tsc_shift);
+ return tsc + cv->tsc_offset;
}
/*
@@ -824,15 +829,17 @@ static inline u64 compute_ref_tsc(int cpu)
u64 kvm_get_ref_tsc(void)
{
int cpu, gen;
+ struct cpu_tsc_vars *cv;
u64 tsc;
cpu = get_cpu();
+ cv = &per_cpu(cpu_tsc_vars, cpu);
again:
- gen = per_cpu(cpu_tsc_generation, cpu);
+ gen = cv->tsc_generation;
smp_rmb();
- tsc = compute_ref_tsc(cpu);
+ tsc = compute_ref_tsc();
smp_rmb();
- if (unlikely(gen != per_cpu(cpu_tsc_generation, cpu)))
+ if (unlikely(gen != cv->tsc_generation))
goto again;
put_cpu();
return tsc;
@@ -889,7 +896,7 @@ static void sync_tsc_helper(int measure_cpu, s64 *delta, atomic_t *ready)
/* wait */;
}
native_cpuid(&junk, &junk, &junk, &junk);
- tsc = compute_ref_tsc(cpu);
+ tsc = compute_ref_tsc();
rdtsc_barrier();
if (cpu == measure_cpu) {
while (!atomic_read(ready))
@@ -972,17 +979,18 @@ static void kvm_sync_tsc(void *cpup)
unsigned long flags;
s64 *delta1, *delta2;
static atomic_t ready ____cacheline_aligned = ATOMIC_INIT(1);
+ struct cpu_tsc_vars *cv = &per_cpu(cpu_tsc_vars, new_cpu);
BUG_ON(tsc_base_cpu == -1);
- pr_debug("%s: IN, cpu = %d, freq = %ldkHz, tsc_base_cpu = %d\n", __func__, raw_smp_processor_id(), per_cpu(cpu_tsc_khz, raw_smp_processor_id()) , tsc_base_cpu);
local_irq_save(flags);
if (raw_smp_processor_id() == new_cpu) {
- per_cpu(cpu_tsc_measure_base, new_cpu) = native_read_tsc();
- per_cpu(cpu_tsc_offset, new_cpu) = 0;
- compute_best_multiplier(ref_tsc_khz,
- per_cpu(cpu_tsc_khz, new_cpu),
- &per_cpu(cpu_tsc_multiplier, new_cpu),
- &per_cpu(cpu_tsc_shift, new_cpu));
+ cv->tsc_measure_base = native_read_tsc();
+ cv->tsc_offset = 0;
+ compute_best_multiplier(ref_tsc_khz, cv->tsc_khz,
+ &cv->tsc_multiplier, &cv->tsc_shift);
+ pr_debug("%s: IN, cpu = %d, freq = %ldkHz, measure_base = %lld,"
+ " tsc_base_cpu = %d\n", __func__, new_cpu, cv->tsc_khz,
+ cv->tsc_measure_base, tsc_base_cpu);
}
delta1 = per_cpu(delta_array, tsc_base_cpu).delta;
delta2 = per_cpu(delta_array, new_cpu).delta;
@@ -1004,22 +1012,32 @@ static void kvm_sync_tsc(void *cpup)
accumulator -= average_samples(&delta2[2], SYNC_TRIES-3);
accumulator /= 2;
- per_cpu(cpu_tsc_offset, new_cpu) = accumulator;
- ++per_cpu(cpu_tsc_generation, new_cpu);
- atomic_set(&per_cpu(cpu_tsc_synchronized, new_cpu), 1);
- pr_debug("%s: OUT, cpu = %d, cpu_tsc_offset = %lld, cpu_tsc_multiplier=%ld, cpu_tsc_shift=%d\n", __func__, raw_smp_processor_id(), per_cpu(cpu_tsc_offset, new_cpu), per_cpu(cpu_tsc_multiplier, new_cpu), per_cpu(cpu_tsc_shift, new_cpu));
+ cv->tsc_offset = accumulator;
+ smp_wmb();
+ ++cv->tsc_generation;
+ atomic_set(&cv->tsc_synchronized, 1);
+ ++kvm_stats.tsc_resync;
+ pr_debug("%s: OUT, cpu = %d, delta = %lld, cpu_tsc_offset = "
+ "%lld, cpu_tsc_multiplier=%ld, cpu_tsc_shift=%d\n",
+ __func__, new_cpu, accumulator, cv->tsc_offset,
+ cv->tsc_multiplier, cv->tsc_shift);
}
local_irq_restore(flags);
}
static void kvm_do_sync_tsc(int cpu)
{
+ BUG_ON(tsc_base_cpu == -1);
spin_lock(&kvm_tsc_lock);
/* tsc_base_cpu can change without tsc_lock, so recheck */
if (unlikely(cpu == tsc_base_cpu))
goto out_unlock;
+ /*
+ * We're calling a co-routine; if we're one of the called CPUs, we
+ * must call ourselves last.
+ */
if (raw_smp_processor_id() != tsc_base_cpu) {
smp_call_function_single(tsc_base_cpu, kvm_sync_tsc,
(void *)&cpu, 0);
@@ -1046,12 +1064,12 @@ static void kvm_write_guest_time(struct kvm_vcpu *v)
if ((!vcpu->time_page))
return;
- this_tsc_khz = get_cpu_var(cpu_tsc_khz);
+ this_tsc_khz = get_cpu_var(cpu_tsc_vars).tsc_khz;
if (unlikely(vcpu->hv_clock_tsc_khz != this_tsc_khz)) {
kvm_set_time_scale(this_tsc_khz, &vcpu->hv_clock);
vcpu->hv_clock_tsc_khz = this_tsc_khz;
}
- put_cpu_var(cpu_tsc_khz);
+ put_cpu_var(cpu_tsc_vars);
/* Keep irq disabled to prevent changes to the clock */
local_irq_save(flags);
@@ -3572,18 +3590,16 @@ static void resync_user(struct work_struct *work)
static void resync(void *info)
{
- int cpu;
+ struct cpu_tsc_vars *cv = &__get_cpu_var(cpu_tsc_vars);
u64 tsc;
/* Fixup our own values to stay in sync with the reference */
- cpu = raw_smp_processor_id();
- tsc = compute_ref_tsc(cpu);
- per_cpu(cpu_tsc_measure_base, cpu) = native_read_tsc();
- per_cpu(cpu_tsc_offset, cpu) = tsc;
- compute_best_multiplier(ref_tsc_khz, per_cpu(cpu_tsc_khz, cpu),
- &per_cpu(cpu_tsc_multiplier, cpu),
- &per_cpu(cpu_tsc_shift, cpu));
- atomic_set(&per_cpu(cpu_tsc_synchronized, cpu), 1);
+ tsc = compute_ref_tsc();
+ cv->tsc_measure_base = native_read_tsc();
+ cv->tsc_offset = tsc;
+ compute_best_multiplier(ref_tsc_khz, cv->tsc_khz, &cv->tsc_multiplier,
+ &cv->tsc_shift);
+ atomic_set(&cv->tsc_synchronized, 1);
/* Then, get everybody else on board */
if (!work_scheduled) {
@@ -3599,6 +3615,7 @@ static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long va
struct kvm *kvm;
struct kvm_vcpu *vcpu;
int i, send_ipi = 0;
+ struct cpu_tsc_vars *cv = &per_cpu(cpu_tsc_vars, freq->cpu);
/*
* There is no way to precisely know the TSC value at which time the
@@ -3612,7 +3629,7 @@ static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long va
*/
if (val == CPUFREQ_PRECHANGE) {
get_online_cpus();
- atomic_set(&per_cpu(cpu_tsc_synchronized, freq->cpu), 0);
+ atomic_set(&cv->tsc_synchronized, 0);
spin_lock(&kvm_lock);
list_for_each_entry(kvm, &vm_list, vm_list) {
kvm_for_each_vcpu(i, vcpu, kvm) {
@@ -3642,7 +3659,7 @@ static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long va
* Just use the vm_tsc_lock for a mutex.
*/
spin_lock(&kvm_tsc_lock);
- per_cpu(cpu_tsc_khz, freq->cpu) = freq->new;
+ cv->tsc_khz = freq->new;
spin_unlock(&kvm_tsc_lock);
return 0;
@@ -3691,12 +3708,13 @@ static int kvm_x86_cpu_hotplug(struct notifier_block *notifier,
case CPU_DYING:
case CPU_UP_CANCELED:
- atomic_set(&per_cpu(cpu_tsc_synchronized, cpu), 0);
+ atomic_set(&per_cpu(cpu_tsc_vars, cpu).tsc_synchronized, 0);
break;
case CPU_ONLINE:
if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
- per_cpu(cpu_tsc_khz, cpu) = cpufreq_quick_get(cpu);
+ per_cpu(cpu_tsc_vars, cpu).tsc_khz =
+ cpufreq_quick_get(cpu);
kvm_do_sync_tsc(cpu);
break;
}
@@ -3767,13 +3785,13 @@ static void kvm_timer_init(void)
if (!khz)
khz = tsc_khz;
spin_lock(&kvm_tsc_lock);
- if (!per_cpu(cpu_tsc_khz, cpu))
- per_cpu(cpu_tsc_khz, cpu) = khz;
+ if (!per_cpu(cpu_tsc_vars, cpu).tsc_khz)
+ per_cpu(cpu_tsc_vars, cpu).tsc_khz = khz;
spin_unlock(&kvm_tsc_lock);
}
} else {
for_each_possible_cpu(cpu)
- per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
+ per_cpu(cpu_tsc_vars, cpu).tsc_khz = tsc_khz;
}
tsc_base_cpu = get_cpu();
resync(NULL);
--
1.6.5.2
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
