virtualization►kvm

Cpu features about kvm hidden

[TOC]

What kvm hidden did to qemu

Based on last blog, we can see how libvirt cpu feature configuration changes qemu cpuid. And we figure out hypervisor disable configuration have what kind of influence.

Then another recommanded feature from libvirt is kvm hidden. In the same way with last blog, we can find libvirt will configure kvm=off to -cpu and according to qemu:

1
2
3
4
5
6
7
8
9
10
11
12
13
DEFINE_PROP_BOOL("hv-relaxed", X86CPU, hyperv_relaxed_timing, false),
DEFINE_PROP_BOOL("hv-vapic", X86CPU, hyperv_vapic, false),
DEFINE_PROP_BOOL("hv-time", X86CPU, hyperv_time, false),
DEFINE_PROP_BOOL("hv-crash", X86CPU, hyperv_crash, false),
DEFINE_PROP_BOOL("hv-reset", X86CPU, hyperv_reset, false),
DEFINE_PROP_BOOL("hv-vpindex", X86CPU, hyperv_vpindex, false),
DEFINE_PROP_BOOL("hv-runtime", X86CPU, hyperv_runtime, false),
DEFINE_PROP_BOOL("hv-synic", X86CPU, hyperv_synic, false),
DEFINE_PROP_BOOL("hv-stimer", X86CPU, hyperv_stimer, false),
DEFINE_PROP_BOOL("hv-frequencies", X86CPU, hyperv_frequencies, false),
DEFINE_PROP_BOOL("check", X86CPU, check_cpuid, true),
DEFINE_PROP_BOOL("enforce", X86CPU, enforce_cpuid, false),
DEFINE_PROP_BOOL("kvm", X86CPU, expose_kvm, true),

those configures are defined by target/i386/cpu.c in variable x86_cpu_properties.

kvm=off will be treated as “kvm” is false and the local variable of this cpu changes expose_kvm to false.

1
2
3
if (!kvm_enabled() || !cpu->expose_kvm) {
    env->features[FEAT_KVM] = 0;
}

x86_cpu_realizefn will invoke x86_cpu_expand_features to expand features from configuration, as a result FEAT_KVM will disable all features after realize features.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
[FEAT_KVM] = {
    .feat_names = {
        "kvmclock", "kvm-nopiodelay", "kvm-mmu", "kvmclock",
        "kvm-asyncpf", "kvm-steal-time", "kvm-pv-eoi", "kvm-pv-unhalt",
        NULL, "kvm-pv-tlb-flush", NULL, NULL,
        NULL, NULL, NULL, NULL,
        NULL, NULL, NULL, NULL,
        NULL, NULL, NULL, NULL,
        "kvmclock-stable-bit", NULL, NULL, NULL,
        NULL, NULL, NULL, NULL,
    },
    .cpuid_eax = KVM_CPUID_FEATURES, .cpuid_reg = R_EAX,
    .tcg_features = TCG_KVM_FEATURES,
},

check its definition, almost all kvm related features is disabled.

Then go ahead to linux kernel arch/x86/include/uapi/asm/kvm_para.h defines those features from cpuid:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/* This CPUID returns a feature bitmap in eax.  Before enabling a particular
 * paravirtualization, the appropriate feature bit should be checked.
 */
#define KVM_CPUID_FEATURES	0x40000001
#define KVM_FEATURE_CLOCKSOURCE		0
#define KVM_FEATURE_NOP_IO_DELAY	1
#define KVM_FEATURE_MMU_OP		2
/* This indicates that the new set of kvmclock msrs
 * are available. The use of 0x11 and 0x12 is deprecated
 */
#define KVM_FEATURE_CLOCKSOURCE2        3
#define KVM_FEATURE_ASYNC_PF		4
#define KVM_FEATURE_STEAL_TIME		5
#define KVM_FEATURE_PV_EOI		6
#define KVM_FEATURE_PV_UNHALT		7

/* The last 8 bits are used to indicate how to interpret the flags field
 * in pvclock structure. If no bits are set, all flags are ignored.
 */
#define KVM_FEATURE_CLOCKSOURCE_STABLE_BIT	24

And before we check all features details let’s check how linux figure kvm feature at first.

For kernel, check kvm by kvm_para_available:

1
2
3
4
bool kvm_para_available(void)
{
	return kvm_cpuid_base() != 0;
}

which will return a kvm based hypervisor by check cpu_has_hypervisor:

1
2
3
4
5
6
7
8
9
10
static noinline uint32_t __kvm_cpuid_base(void)
{
	if (boot_cpu_data.cpuid_level < 0)
		return 0;	/* So we don't blow up on old processors */

	if (cpu_has_hypervisor)
		return hypervisor_cpuid_base("KVMKVMKVM\0\0\0", 0);

	return 0;
}

and cpu_has_hypervisor is defined from the hypervisor feature we mentioned in last post:

1
#define cpu_has_hypervisor	boot_cpu_has(X86_FEATURE_HYPERVISOR)

So we combine those two part together to check the influence introduced by kvm hidden.

Note: here is the brief description about those features in cpuid:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
function: define KVM_CPUID_FEATURES (0x40000001)
returns : ebx, ecx, edx = 0
          eax = and OR'ed group of (1 << flag), where each flags is:


flag                               || value || meaning
=============================================================================
KVM_FEATURE_CLOCKSOURCE            ||     0 || kvmclock available at msrs
                                   ||       || 0x11 and 0x12.
------------------------------------------------------------------------------
KVM_FEATURE_NOP_IO_DELAY           ||     1 || not necessary to perform delays
                                   ||       || on PIO operations.
------------------------------------------------------------------------------
KVM_FEATURE_MMU_OP                 ||     2 || deprecated.
------------------------------------------------------------------------------
KVM_FEATURE_CLOCKSOURCE2           ||     3 || kvmclock available at msrs
                                   ||       || 0x4b564d00 and 0x4b564d01
------------------------------------------------------------------------------
KVM_FEATURE_ASYNC_PF               ||     4 || async pf can be enabled by
                                   ||       || writing to msr 0x4b564d02
------------------------------------------------------------------------------
KVM_FEATURE_STEAL_TIME             ||     5 || steal time can be enabled by
                                   ||       || writing to msr 0x4b564d03.
------------------------------------------------------------------------------
KVM_FEATURE_PV_EOI                 ||     6 || paravirtualized end of interrupt
                                   ||       || handler can be enabled by writing
                                   ||       || to msr 0x4b564d04.
------------------------------------------------------------------------------
KVM_FEATURE_PV_UNHALT              ||     7 || guest checks this feature bit
                                   ||       || before enabling paravirtualized
                                   ||       || spinlock support.
------------------------------------------------------------------------------
KVM_FEATURE_CLOCKSOURCE_STABLE_BIT ||    24 || host will warn if no guest-side
                                   ||       || per-cpu warps are expected in
                                   ||       || kvmclock.
------------------------------------------------------------------------------

KVM_FEATURE_CLOCKSOURCE & KVM_FEATURE_CLOCKSOURCE2

This feature is used directly when implement kvmclock_init:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
void __init kvmclock_init(void)
{
	struct pvclock_vcpu_time_info *vcpu_time;
	unsigned long mem, mem_wall_clock;
	int size, cpu, wall_clock_size;
	u8 flags;

	size = PAGE_ALIGN(sizeof(struct pvclock_vsyscall_time_info)*NR_CPUS);

	if (!kvm_para_available())
		return;

	if (kvmclock && kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE2)) {
		msr_kvm_system_time = MSR_KVM_SYSTEM_TIME_NEW;
		msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK_NEW;
	} else if (!(kvmclock && kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE)))
		return;

KVM_FEATURE_NOP_IO_DELAY

During guest init, paravirt_ops_setup will use this feature:

1
2
3
4
5
6
7
8
void __init kvm_guest_init(void)
{
	int i;

	if (!kvm_para_available())
		return;

	paravirt_ops_setup();

which changes io_delay of paravirt cpu ops to kvm_io_delay:

1
2
3
4
5
6
7
8
9
10
11
12
static void __init paravirt_ops_setup(void)
{
	pv_info.name = "KVM";
	pv_info.paravirt_enabled = 1;

	if (kvm_para_has_feature(KVM_FEATURE_NOP_IO_DELAY))
		pv_cpu_ops.io_delay = kvm_io_delay;

#ifdef CONFIG_X86_IO_APIC
	no_timer_check = 1;
#endif
}

which just means without any io delay:

1
2
3
4
5
6
/*
 * No need for any "IO delay" on KVM
 */
static void kvm_io_delay(void)
{
}

KVM_FEATURE_MMU_OP

Deprecated.

KVM_FEATURE_ASYNC_PF

When init kvm guest:

1
2
3
4
5
void __init kvm_guest_init(void)
{
  // ...
	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF))
		x86_init.irqs.trap_init = kvm_apf_trap_init;

kvm_apf_trap_init will be set to x86_init.irqs.trap_init which will set async_page_fault when interrupt request for trap operations:

1
2
3
4
static void __init kvm_apf_trap_init(void)
{
	set_intr_gate(14, async_page_fault);
}

And then when init kvm guest cpu, will manually enable cpu to allow to write async page fault:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
static void kvm_guest_cpu_init(void)
{
	if (!kvm_para_available())
		return;

	if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF) && kvmapf) {
		u64 pa = slow_virt_to_phys(this_cpu_ptr(&apf_reason));

#ifdef CONFIG_PREEMPT
		pa |= KVM_ASYNC_PF_SEND_ALWAYS;
#endif
		wrmsrl(MSR_KVM_ASYNC_PF_EN, pa | KVM_ASYNC_PF_ENABLED);
		__this_cpu_write(apf_reason.enabled, 1);
		printk(KERN_INFO"KVM setup async PF for cpu %d\n",
		       smp_processor_id());
	}

Then feature will enable async PF for this cpu.

Note: trap initialize will be done by arch/x86/kernel/traps.c:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
void __init trap_init(void)
{
	int i;

#ifdef CONFIG_EISA
	void __iomem *p = early_ioremap(0x0FFFD9, 4);

	if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24))
		EISA_bus = 1;
	early_iounmap(p, 4);
#endif

	set_intr_gate(X86_TRAP_DE, divide_error);
	set_intr_gate_ist(X86_TRAP_NMI, &nmi, NMI_STACK);
	/* int4 can be called from all */
	set_system_intr_gate(X86_TRAP_OF, &overflow);
	set_intr_gate(X86_TRAP_BR, bounds);
	set_intr_gate(X86_TRAP_UD, invalid_op);
	set_intr_gate(X86_TRAP_NM, device_not_available);
#ifdef CONFIG_X86_32
	set_task_gate(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS);
#else
	set_intr_gate_ist(X86_TRAP_DF, &double_fault, DOUBLEFAULT_STACK);
#endif
	set_intr_gate(X86_TRAP_OLD_MF, coprocessor_segment_overrun);
	set_intr_gate(X86_TRAP_TS, invalid_TSS);
	set_intr_gate(X86_TRAP_NP, segment_not_present);
	set_intr_gate(X86_TRAP_SS, stack_segment);
	set_intr_gate(X86_TRAP_GP, general_protection);
	set_intr_gate(X86_TRAP_SPURIOUS, spurious_interrupt_bug);
	set_intr_gate(X86_TRAP_MF, coprocessor_error);
	set_intr_gate(X86_TRAP_AC, alignment_check);
#ifdef CONFIG_X86_MCE
	set_intr_gate_ist(X86_TRAP_MC, &machine_check, MCE_STACK);
#endif
	set_intr_gate(X86_TRAP_XF, simd_coprocessor_error);

	/* Reserve all the builtin and the syscall vector: */
	for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++)
		set_bit(i, used_vectors);

#ifdef CONFIG_IA32_EMULATION
	set_system_intr_gate(IA32_SYSCALL_VECTOR, ia32_syscall);
	set_bit(IA32_SYSCALL_VECTOR, used_vectors);
#endif

#ifdef CONFIG_X86_32
	set_system_trap_gate(SYSCALL_VECTOR, &system_call);
	set_bit(SYSCALL_VECTOR, used_vectors);
#endif

	/*
	 * Set the IDT descriptor to a fixed read-only location, so that the
	 * "sidt" instruction will not leak the location of the kernel, and
	 * to defend the IDT against arbitrary memory write vulnerabilities.
	 * It will be reloaded in cpu_init() */
	__set_fixmap(FIX_RO_IDT, __pa_symbol(idt_table), PAGE_KERNEL_RO);
	idt_descr.address = fix_to_virt(FIX_RO_IDT);

	/*
	 * Should be a barrier for any external CPU state:
	 */
	cpu_init();

	x86_init.irqs.trap_init();

#ifdef CONFIG_X86_64
	memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16);
	set_nmi_gate(X86_TRAP_DB, &debug);
	set_nmi_gate(X86_TRAP_BP, &int3);
#endif
}

and x86_init.irqs.trap_init(); will be used post other features.

KVM_FEATURE_STEAL_TIME

when do kvm guest init:

1
2
3
4
if (kvm_para_has_feature(KVM_FEATURE_STEAL_TIME)) {
	has_steal_clock = 1;
	pv_time_ops.steal_clock = kvm_steal_clock;
}

Paravirt steal lock will be replaced by kvm

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
static u64 kvm_steal_clock(int cpu)
{
	u64 steal;
	struct kvm_steal_time *src;
	int version;

	src = &per_cpu(steal_time, cpu);
	do {
		version = src->version;
		rmb();
		steal = src->steal;
		rmb();
	} while ((version & 1) || (version != src->version));

	return steal;
}

which will steal the time from cpu directly.

KVM_FEATURE_PV_EOI

From kvm guest init:

1
2
if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
	apic_set_eoi_write(kvm_guest_apic_eoi_write);

During kvm guest cpu init:

1
2
3
4
5
6
7
8
9
if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) {
	unsigned long pa;
	/* Size alignment is implied but just to make it explicit. */
	BUILD_BUG_ON(__alignof__(kvm_apic_eoi) < 4);
	__this_cpu_write(kvm_apic_eoi, 0);
	pa = slow_virt_to_phys(this_cpu_ptr(&kvm_apic_eoi))
		| KVM_MSR_ENABLED;
	wrmsrl(MSR_KVM_PV_EOI_EN, pa);
}

Besides, those paravirt kvm features is used by kernel so those features need to be disabled if kernel changed, for example, load kernel by kexec, to avoid the features pointing to old memory of old kernel, those features will disabled by write msr manually:

1
2
3
4
5
6
7
8
9
10
11
12
static void kvm_pv_guest_cpu_reboot(void *unused)
{
	/*
	 * We disable PV EOI before we load a new kernel by kexec,
	 * since MSR_KVM_PV_EOI_EN stores a pointer into old kernel's memory.
	 * New kernel can re-enable when it boots.
	 */
	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
	kvm_pv_disable_apf();
	kvm_disable_steal_time();
}

So does kvm guest cpu offline do:

1
2
3
4
5
6
7
8
static void kvm_guest_cpu_offline(void *dummy)
{
	kvm_disable_steal_time();
	if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
		wrmsrl(MSR_KVM_PV_EOI_EN, 0);
	kvm_pv_disable_apf();
	apf_task_wake_all();
}

That’s all due to paravirt use shared memory to use those features between guest and host.

KVM_FEATURE_PV_UNHALT

Allow to use para-virtualized spinlock

1
2
3
4
5
6
7
8
void __init kvm_spinlock_init(void)
{
	if (!kvm_para_available())
		return;
	/* Does host kernel support KVM_FEATURE_PV_UNHALT? */
	if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
		return;

KVM_FEATURE_CLOCKSOURCE_STABLE_BIT

kvm clock will set a PVCLOCK_TSC_STABLE_BIT to pvclock.

1
2
3
4
5
printk(KERN_INFO "kvm-clock: Using msrs %x and %x",
       msr_kvm_system_time, msr_kvm_wall_clock);

if (kvm_para_has_feature(KVM_FEATURE_CLOCKSOURCE_STABLE_BIT))
	pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);

when stable source detected:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
{
	unsigned version;
	u64 ret;
	u64 last;
	u8 flags;

	do {
		version = pvclock_read_begin(src);
		ret = __pvclock_read_cycles(src, rdtsc_ordered());
		flags = src->flags;
	} while (pvclock_read_retry(src, version));

	if (unlikely((flags & PVCLOCK_GUEST_STOPPED) != 0)) {
		src->flags &= ~PVCLOCK_GUEST_STOPPED;
		pvclock_touch_watchdogs();
	}

	if ((valid_flags & PVCLOCK_TSC_STABLE_BIT) &&
		(flags & PVCLOCK_TSC_STABLE_BIT))
		return ret;

clocksource read will return directly.

Hyper-v impact

linux will converting hyperv and kvmclock

1
2
3
4
5
6
7
static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock,
					HV_REFERENCE_TSC_PAGE *tsc_ref)
{
	u64 max_mul;

	if (!(hv_clock->flags & PVCLOCK_TSC_STABLE_BIT))
		return false;

but if no stable tsc allowed, hypervclock and kvmclock computing will be skipped.

Function chain as following:

kvm_guest_time_update -> kvm_hv_setup_tsc_page -> compute_tsc_page_parameters

And source is from kvm request:

1
2
3
4
5
if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
	r = kvm_guest_time_update(vcpu);
	if (unlikely(r))
		goto out;
}

We need to know more about KVM_REQ_CLOCK_UPDATE to figure out when. this request will be used.

The clue is kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); make request usage.

  • Ioctl kvm clock set -> KVM_SET_CLOCK -> kvm_gen_update_masterclock

  • kvm_check_request(KVM_REQ_MASTERCLOCK_UPDATE, vcpu) -> kvm_gen_update_masterclock

  • kvm_guest_time_update -> kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
    first update is from kvm request:

    1
    2
    3
    4
    5
    if (kvm_check_request(KVM_REQ_CLOCK_UPDATE, vcpu)) {
    	r = kvm_guest_time_update(vcpu);
    	if (unlikely(r))
    		goto out;
    }

    then interrupt will be disabled to prevent clock changes:

    1
    2
    3
    4
    5
    6
    7
    8
    /* Keep irq disabled to prevent changes to the clock */
    local_irq_save(flags);
    this_tsc_khz = __this_cpu_read(cpu_tsc_khz);
    if (unlikely(this_tsc_khz == 0)) {
    	local_irq_restore(flags);
    	kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
    	return 1;
    }

  • INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn); -> kvmclock_update_fn -> kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);
    kvm lock will be updated by a schedule:

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    /*
     * kvmclock updates which are isolated to a given vcpu, such as
     * vcpu->cpu migration, should not allow system_timestamp from
     * the rest of the vcpus to remain static. Otherwise ntp frequency
     * correction applies to one vcpu's system_timestamp but not
     * the others.
     *
     * So in those cases, request a kvmclock update for all vcpus.
     * We need to rate-limit these requests though, as they can
     * considerably slow guests that have a large number of vcpus.
     * The time for a remote vcpu to update its kvmclock is bound
     * by the delay we use to rate-limit the updates.
     */

    #define KVMCLOCK_UPDATE_DELAY msecs_to_jiffies(100)

    and kvmlock sync delays are

    1
    #define KVMCLOCK_SYNC_PERIOD (300 * HZ)

  • kvm_check_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu) -> kvm_gen_kvmclock_update -> kvm_make_request(KVM_REQ_CLOCK_UPDATE, v);

    • MSR_KVM_SYSTEM_TIME

    • kvm_arch_vcpu_load
      update clock if no master clock or host cpu to sync.

      1
      2
      3
      4
      5
      6
      7
      8
      9
      /*
       * On a host with synchronized TSC, there is no need to update
       * kvmclock on vcpu->cpu migration
       */
      if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
      	kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
      if (vcpu->cpu != cpu)
      	kvm_migrate_timers(vcpu);
      vcpu->cpu = cpu;

  • kvm_arch_vcpu_load -> kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
    Adjust time if needed

    1
    2
    3
    4
    5
    6
    /* Apply any externally detected TSC adjustments (due to suspend) */
    if (unlikely(vcpu->arch.tsc_offset_adjustment)) {
    	adjust_tsc_offset_host(vcpu, vcpu->arch.tsc_offset_adjustment);
    	vcpu->arch.tsc_offset_adjustment = 0;
    	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
    }

  • kvm_set_guest_paused -> kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
    if guest kernel stopped by hypervisor use this to update pv clock.

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    /*
     * kvm_set_guest_paused() indicates to the guest kernel that it has been
     * stopped by the hypervisor.  This function will be called from the host only.
     * EINVAL is returned when the host attempts to set the flag for a guest that
     * does not support pv clocks.
     */
    static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
    {
    	if (!vcpu->arch.pv_time_enabled)
    		return -EINVAL;
    	vcpu->arch.pvclock_set_guest_stopped_request = true;
    	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
    	return 0;
    }

  • kvmclock_cpufreq_notifier -> kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
    see the annotation from code:

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    /*
     * We allow guests to temporarily run on slowing clocks,
     * provided we notify them after, or to run on accelerating
     * clocks, provided we notify them before.  Thus time never
     * goes backwards.
     *
     * However, we have a problem.  We can't atomically update
     * the frequency of a given CPU from this function; it is
     * merely a notifier, which can be called from any CPU.
     * Changing the TSC frequency at arbitrary points in time
     * requires a recomputation of local variables related to
     * the TSC for each VCPU.  We must flag these local variables
     * to be updated and be sure the update takes place with the
     * new frequency before any guests proceed.
     *
     * Unfortunately, the combination of hotplug CPU and frequency
     * change creates an intractable locking scenario; the order
     * of when these callouts happen is undefined with respect to
     * CPU hotplug, and they can race with each other.  As such,
     * merely setting per_cpu(cpu_tsc_khz) = X during a hotadd is
     * undefined; you can actually have a CPU frequency change take
     * place in between the computation of X and the setting of the
     * variable.  To protect against this problem, all updates of
     * the per_cpu tsc_khz variable are done in an interrupt
     * protected IPI, and all callers wishing to update the value
     * must wait for a synchronous IPI to complete (which is trivial
     * if the caller is on the CPU already).  This establishes the
     * necessary total order on variable updates.
     *
     * Note that because a guest time update may take place
     * anytime after the setting of the VCPU's request bit, the
     * correct TSC value must be set before the request.  However,
     * to ensure the update actually makes it to any guest which
     * starts running in hardware virtualization between the set
     * and the acquisition of the spinlock, we must also ping the
     * CPU after setting the request bit.
     *
     */

  • after kvm_guest_exit();
    update clock if vcpu request clock always up to date.

    1
    2
    if (unlikely(vcpu->arch.tsc_always_catchup))
    	kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);

  • hardware_enable_nolock -> kvm_arch_hardware_enable -> kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
    multi functino access hardware_enable_nolock

    • kvm_cpu_hotplug
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
    			   void *v)
    {
    	val &= ~CPU_TASKS_FROZEN;
    	switch (val) {
    	case CPU_DYING:
    		hardware_disable();
    		break;
    	case CPU_STARTING:
    		hardware_enable();
    		break;
    	}
    	return NOTIFY_OK;
    }
    • kvm_resume

Note: for hv_stimer 

1
2
3
4
5
6
7
/*
 * KVM_REQ_HV_STIMER has to be processed after
 * KVM_REQ_CLOCK_UPDATE, because Hyper-V SynIC timers
 * depend on the guest clock being up-to-date
 */
if (kvm_check_request(KVM_REQ_HV_STIMER, vcpu))
	kvm_hv_process_stimers(vcpu);

will be done after guest clock up-to-date.

Hyper-v impact conclusion

With kvm hidden, hyper-v tsc compute will be skipped:

1
2
3
4
5
6
7
static bool compute_tsc_page_parameters(struct pvclock_vcpu_time_info *hv_clock,
					struct ms_hyperv_tsc_page *tsc_ref)
{
	u64 max_mul;

	if (!(hv_clock->flags & PVCLOCK_TSC_STABLE_BIT))
		return false;

which can be triggered by above kvm code.

During migration, we know that guest will be stopped (paused) by KVM_KVMCLOCK_CTRL and we could check kvm userspace’s (qemu) usage:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
static void kvmclock_vm_state_change(void *opaque, int running,
                                     RunState state)
{
    KVMClockState *s = opaque;
    CPUState *cpu;
    int cap_clock_ctrl = kvm_check_extension(kvm_state, KVM_CAP_KVMCLOCK_CTRL);
    int ret;

    if (running) {
        struct kvm_clock_data data = {};

        /*
         * If the host where s->clock was read did not support reliable
         * KVM_GET_CLOCK, read kvmclock value from memory.
         */
        if (!s->clock_is_reliable) {
            uint64_t pvclock_via_mem = kvmclock_current_nsec(s);
            /* We can't rely on the saved clock value, just discard it */
            if (pvclock_via_mem) {
                s->clock = pvclock_via_mem;
            }
        }

        s->clock_valid = false;

        data.clock = s->clock;
        ret = kvm_vm_ioctl(kvm_state, KVM_SET_CLOCK, &data);
        if (ret < 0) {
            fprintf(stderr, "KVM_SET_CLOCK failed: %s\n", strerror(ret));
            abort();
        }

        if (!cap_clock_ctrl) {
            return;
        }
        CPU_FOREACH(cpu) {
            run_on_cpu(cpu, do_kvmclock_ctrl, RUN_ON_CPU_NULL);
        }
    } else {

        if (s->clock_valid) {
            return;
        }

        s->runstate_paused = runstate_check(RUN_STATE_PAUSED);

        kvm_synchronize_all_tsc();

        kvm_update_clock(s);
        /*
         * If the VM is stopped, declare the clock state valid to
         * avoid re-reading it on next vmsave (which would return
         * a different value). Will be reset when the VM is continued.
         */
        s->clock_valid = true;
    }
}

when set guest to running, qemu will use KVM_SET_CLOCK else will use kvm_update_clock works as following:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
static void kvm_update_clock(KVMClockState *s)
{
    struct kvm_clock_data data;
    int ret;

    ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
    if (ret < 0) {
        fprintf(stderr, "KVM_GET_CLOCK failed: %s\n", strerror(ret));
                abort();
    }
    s->clock = data.clock;

    /* If kvm_has_adjust_clock_stable() is false, KVM_GET_CLOCK returns
     * essentially CLOCK_MONOTONIC plus a guest-specific adjustment.  This
     * can drift from the TSC-based value that is computed by the guest,
     * so we need to go through kvmclock_current_nsec().  If
     * kvm_has_adjust_clock_stable() is true, and the flags contain
     * KVM_CLOCK_TSC_STABLE, then KVM_GET_CLOCK returns a TSC-based value
     * and kvmclock_current_nsec() is not necessary.
     *
     * Here, however, we need not check KVM_CLOCK_TSC_STABLE.  This is because:
     *
     * - if the host has disabled the kvmclock master clock, the guest already
     *   has protection against time going backwards.  This "safety net" is only
     *   absent when kvmclock is stable;
     *
     * - therefore, we can replace a check like
     *
     *       if last KVM_GET_CLOCK was not reliable then
     *               read from memory
     *
     *   with
     *
     *       if last KVM_GET_CLOCK was not reliable && masterclock is enabled
     *               read from memory
     *
     * However:
     *
     * - if kvm_has_adjust_clock_stable() returns false, the left side is
     *   always true (KVM_GET_CLOCK is never reliable), and the right side is
     *   unknown (because we don't have data.flags).  We must assume it's true
     *   and read from memory.
     *
     * - if kvm_has_adjust_clock_stable() returns true, the result of the &&
     *   is always false (masterclock is enabled iff KVM_GET_CLOCK is reliable)
     *
     * So we can just use this instead:
     *
     *       if !kvm_has_adjust_clock_stable() then
     *               read from memory
     */
    s->clock_is_reliable = kvm_has_adjust_clock_stable();
}

But from the annotation in kvmclock_vm_state_change:

1
2
3
4
5
/*
 * If the VM is stopped, declare the clock state valid to
 * avoid re-reading it on next vmsave (which would return
 * a different value). Will be reset when the VM is continued.
 */

qemu seems to relay on vmsave to reset the guest while vm is continued, we just keep our eyes on that.

Combine qemu guest state change hook:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
case KVM_SET_CLOCK: {
	struct kvm_arch *ka = &kvm->arch;
	struct kvm_clock_data user_ns;
	u64 now_ns;

	r = -EFAULT;
	if (copy_from_user(&user_ns, argp, sizeof(user_ns)))
		goto out;

	r = -EINVAL;
	if (user_ns.flags)
		goto out;

	r = 0;
	/*
	 * TODO: userspace has to take care of races with VCPU_RUN, so
	 * kvm_gen_update_masterclock() can be cut down to locked
	 * pvclock_update_vm_gtod_copy().
	 */
	kvm_gen_update_masterclock(kvm);

	/*
	 * This pairs with kvm_guest_time_update(): when masterclock is
	 * in use, we use master_kernel_ns + kvmclock_offset to set
	 * unsigned 'system_time' so if we use get_kvmclock_ns() (which
	 * is slightly ahead) here we risk going negative on unsigned
	 * 'system_time' when 'user_ns.clock' is very small.
	 */
	spin_lock_irq(&ka->pvclock_gtod_sync_lock);
	if (kvm->arch.use_master_clock)
		now_ns = ka->master_kernel_ns;
	else
		now_ns = get_kvmclock_base_ns();
	ka->kvmclock_offset = user_ns.clock - now_ns;
	spin_unlock_irq(&ka->pvclock_gtod_sync_lock);

	kvm_make_all_cpus_request(kvm, KVM_REQ_CLOCK_UPDATE);

will be used to update guest clock.

Hand on test to confirm clock updates

Enable kvm trace by:

1
echo 1 > /sys/kernel/debug/tracing/events/kvm/enable

Then collect the output when vm migrated to this host:

1
cat /sys/kernel/debug/tracing/trace_pipe > trace_migrated_vm

We can see following logs at first:

1
2
3
4
5
6
7
8
9
<...>-89383 [001] .... 97852.765277: kvm_update_master_clock: masterclock 0 hostclock 0x2 offsetmatched 0
<...>-89441 [002] d... 97852.785366: kvm_write_tsc_offset: vcpu=0 prev=0 next=18446539041810541506
<...>-89441 [002] d... 97852.785402: kvm_track_tsc: vcpu_id 0 masterclock 0 offsetmatched 0 nr_online 1 hostclock 0x2
<...>-89442 [002] d... 97852.786522: kvm_write_tsc_offset: vcpu=1 prev=0 next=18446539041810541506
<...>-89442 [002] d... 97852.786533: kvm_track_tsc: vcpu_id 1 masterclock 0 offsetmatched 1 nr_online 2 hostclock 0x2
<...>-89443 [002] d... 97852.787341: kvm_write_tsc_offset: vcpu=2 prev=0 next=18446539041810541506
<...>-89443 [002] d... 97852.787348: kvm_track_tsc: vcpu_id 2 masterclock 0 offsetmatched 2 nr_online 3 hostclock 0x2
<...>-89444 [002] d... 97852.788099: kvm_write_tsc_offset: vcpu=3 prev=0 next=18446539041810541506
<...>-89444 [002] d... 97852.788120: kvm_track_tsc: vcpu_id 3 masterclock 0 offsetmatched 3 nr_online 4 hostclock 0x2

kvm_update_master_clock is used for vm migration:

And the tsc offset changed:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
<...>-89441 [002] d... 97852.785366: kvm_write_tsc_offset: vcpu=0 prev=0 next=18446539041810541506
<...>-89441 [002] d... 97852.785402: kvm_track_tsc: vcpu_id 0 masterclock 0 offsetmatched 0 nr_online 1 hostclock 0x2
<...>-89442 [002] d... 97852.786522: kvm_write_tsc_offset: vcpu=1 prev=0 next=18446539041810541506
<...>-89442 [002] d... 97852.786533: kvm_track_tsc: vcpu_id 1 masterclock 0 offsetmatched 1 nr_online 2 hostclock 0x2
<...>-89443 [002] d... 97852.787341: kvm_write_tsc_offset: vcpu=2 prev=0 next=18446539041810541506
<...>-89443 [002] d... 97852.787348: kvm_track_tsc: vcpu_id 2 masterclock 0 offsetmatched 2 nr_online 3 hostclock 0x2
<...>-89444 [002] d... 97852.788099: kvm_write_tsc_offset: vcpu=3 prev=0 next=18446539041810541506

<...>-89441 [003] d... 97852.872014: kvm_write_tsc_offset: vcpu=0 prev=18446539041810541506 next=18446539041810541506
<...>-89442 [003] d... 97852.872105: kvm_write_tsc_offset: vcpu=1 prev=18446539041810541506 next=18446539041810541506
<...>-89443 [003] d... 97852.872189: kvm_write_tsc_offset: vcpu=2 prev=18446539041810541506 next=18446539041810541506
<...>-89444 [003] d... 97852.872264: kvm_write_tsc_offset: vcpu=3 prev=18446539041810541506 next=18446539041810541506

<...>-89441 [000] d... 97856.399432: kvm_write_tsc_offset: vcpu=0 prev=18446539041810541506 next=18446562414330701094
<...>-89442 [000] d... 97856.403066: kvm_write_tsc_offset: vcpu=1 prev=18446539041810541506 next=18446562414330701094
<...>-89443 [000] d... 97856.403273: kvm_write_tsc_offset: vcpu=2 prev=18446539041810541506 next=18446562414330701094
<...>-89444 [000] d... 97856.403414: kvm_write_tsc_offset: vcpu=3 prev=18446539041810541506 next=18446562414330701094

Follow the trace we can find linux kernel code:

kvm_vcpu_write_tsc_offset -> kvm_x86_write_l1_tsc_offset -> write_l1_tsc_offset -> vmx_write_l1_tsc_offset -> trace_kvm_write_tsc_offset

And there are multi usages of kvm_vcpu_write_tsc_offset

  • kvm_synchronize_tsc
    • MSR_IA32_TSC -> kvm_synchronize_tsc
    • kvm_vm_ioctl_create_vcpu -> kvm_arch_vcpu_postcreate -> kvm_synchronize_tsc
  • adjust_tsc_offset_guest
    • kvm_guest_time_update -> adjust_tsc_offset_guest and kvm_hv_setup_tsc_page this is hyper-v impacted case
    • MSR_IA32_TSC -> adjust_tsc_offset_guest
    • MSR_IA32_TSC_ADJUST -> adjust_tsc_offset_guest
    • kvm_arch_vcpu_load -> adjust_tsc_offset_host -> adjust_tsc_offset_guest
  • kvm_arch_vcpu_load same as above

So the following three parts of kvm_vcpu_write_tsc_offset matches with guest creation.

  • Create vcpu
  • Load vcpu
  • Adjust tsc offset

In last guest hang post, we can see windows guest try to get counter ref:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
static u64 get_time_ref_counter(struct kvm *kvm)
{
	struct kvm_hv *hv = to_kvm_hv(kvm);
	struct kvm_vcpu *vcpu;
	u64 tsc;

	/*
	 * Fall back to get_kvmclock_ns() when TSC page hasn't been set up,
	 * is broken, disabled or being updated.
	 */
	if (hv->hv_tsc_page_status != HV_TSC_PAGE_SET)
		return div_u64(get_kvmclock_ns(kvm), 100);

	vcpu = kvm_get_vcpu(kvm, 0);
	tsc = kvm_read_l1_tsc(vcpu, rdtsc());
	return mul_u64_u64_shr(tsc, hv->tsc_ref.tsc_scale, 64)
		+ hv->tsc_ref.tsc_offset;
}

But this is used by MSR read request from guest. And now we need to debug hv_tsc_page_status and kvm_hv_setup_tsc_page usage.

Without kvm hidden:

1
2
3
4
5
<...>-114210 [002] d... 12255.411580: kvm_exit: vcpu 1 reason MSR_READ rip 0xfffff800ece454c5 info1 0x0000000000000000 info2 0x0000000000000000 intr_info 0x00000000 error_code 0x00000000
<...>-114210 [002] .... 12255.411581: kvm_msr: msr_read 40000020 = 0x6fac3c27
<...>-114210 [002] d... 12255.411582: kvm_entry: vcpu 1, rip 0xfffff800ece454c7
<...>-114211 [000] .... 12255.411585: kvm_vcpu_wakeup: wait time 1759974 ns, polling valid
<...>-114211 [000] .... 12255.411585: kvm_hv_timer_state: vcpu_id 2 hv_timer 0

We can find kvm_hv_timer_state in trace, and according to linux kernel code:

1
2
TRACE_EVENT(kvm_hv_timer_state,
		TP_PROTO(unsigned int vcpu_id, unsigned int hv_timer_in_use),

There are two ways to show the trace:

  • start_sw_timer -> trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false); which is always false (means 0 in trace)
  • start_hv_timer -> trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use); which returns hv_timer_in_use from ktimer->hv_timer_in_use

Check the code about start_hv_timer:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
static bool start_hv_timer(struct kvm_lapic *apic)
{
	struct kvm_timer *ktimer = &apic->lapic_timer;
	struct kvm_vcpu *vcpu = apic->vcpu;
	bool expired;

	WARN_ON(preemptible());
	if (!kvm_can_use_hv_timer(vcpu))
		return false;

	if (!ktimer->tscdeadline)
		return false;

	if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))
		return false;

	ktimer->hv_timer_in_use = true;
	hrtimer_cancel(&ktimer->timer);

	/*
	 * To simplify handling the periodic timer, leave the hv timer running
	 * even if the deadline timer has expired, i.e. rely on the resulting
	 * VM-Exit to recompute the periodic timer's target expiration.
	 */
	if (!apic_lvtt_period(apic)) {
		/*
		 * Cancel the hv timer if the sw timer fired while the hv timer
		 * was being programmed, or if the hv timer itself expired.
		 */
		if (atomic_read(&ktimer->pending)) {
			cancel_hv_timer(apic);
		} else if (expired) {
			apic_timer_expired(apic, false);
			cancel_hv_timer(apic);
		}
	}

	trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);

	return true;
}

ktimer->hv_timer_in_use is set to true so we focus on start_sw_timer next.

There are several ways to goes into restart_apic_timer

  • restart_apic_timer -> start_sw_timer
    • vmx_exit_handlers_fastpath or __vmx_handle_exit -> handle_fastpath_preemption_timer -> kvm_lapic_expired_hv_timer -> restart_apic_timer
    • vcpu_block -> post_block -> vmx_post_block -> kvm_lapic_switch_to_hv_timer -> restart_apic_timer
    • MSR_IA32_TSC_DEADLINE ->handle_fastpath_set_tscdeadline -> kvm_set_lapic_tscdeadline_msr -> __start_apic_timer -> restart_apic_timer
    • APIC_TDCR -> restart_apic_timer
  • vcpu_block -> vmx_pre_block -> kvm_lapic_switch_to_sw_timer -> start_sw_timer

Because we see a trace before shows:

1
kvm_vcpu_wakeup: wait time 1759974 ns, polling valid

which is in kvm_vcpu_block, so this means vmx_post_block restart_apic_timer 

1
2
trace_kvm_vcpu_wakeup(block_ns, waited, vcpu_valid_wakeup(vcpu));
kvm_arch_vcpu_block_finish(vcpu);

And because the code runs as:

1
2
if (!start_hv_timer(apic))
	start_sw_timer(apic);

start_hv_timer must returns false:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
static bool start_hv_timer(struct kvm_lapic *apic)
{
	struct kvm_timer *ktimer = &apic->lapic_timer;
	struct kvm_vcpu *vcpu = apic->vcpu;
	bool expired;

	WARN_ON(preemptible());
	if (!kvm_can_use_hv_timer(vcpu))
		return false;

	if (!ktimer->tscdeadline)
		return false;

	if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))
		return false;

kvm_can_use_hv_timer check seems works on x86 machine and while X86_FEATURE_MWAIT is supported.

From the trace we could know, when vcpu exit and come back to work, the timer will be updated, and use vcpu 3 as example:

1
<...>-114212 [002] d... 12297.437890: kvm_exit: vcpu 3 reason HLT rip 0xfffff800ecc2b36e info1 0x0000000000000000 info2 0x0000000000000000 intr_info 0x00000000 error_code 0x00000000

vcpu 3 HLT and cause kvm_exit.

Then it wakeup after 4774180 ns and hv_timer is traced without usage.

1
2
<...>-114212 [002] .... 12255.393408: kvm_vcpu_wakeup: wait time 4774180 ns, polling valid
<...>-114212 [002] .... 12255.393410: kvm_hv_timer_state: vcpu_id 3 hv_timer 0

And hv_timer will be cancelled after live migration:

1
2
if (apic->lapic_timer.hv_timer_in_use)
	cancel_hv_timer(apic);

Let’s check hv_timer before migration:

Can we resolve compatibility issues?

See the code of qemu, it will disable features of FEAT_KVM after all features setup, so we can not manually assign those features:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
for (l = plus_features; l; l = l->next) {
    const char *prop = l->data;
    object_property_set_bool(OBJECT(cpu), true, prop, &local_err);
    if (local_err) {
        goto out;
    }
}

for (l = minus_features; l; l = l->next) {
    const char *prop = l->data;
    object_property_set_bool(OBJECT(cpu), false, prop, &local_err);
    if (local_err) {
        goto out;
    }
}

if (!kvm_enabled() || !cpu->expose_kvm) {
    env->features[FEAT_KVM] = 0;
}
virtualization►translation►virtio

Virtio on Linux

Introduction

Virtio is an open standard that defines a protocol for communication between drivers and devices of different types, see Chapter 5 (“Device Types”) of the virtio spec ([1]). Originally developed as a standard for paravirtualized devices implemented by a hypervisor, it can be used to interface any compliant device (real or emulated) with a driver.

Virtio是一个开放的标准，它定义了驱动程序和不同类型的设备之间的通信协议，见virtio规范（[1]）的第五章（”设备类型”）。它最初是作为由管理程序实现的准虚拟化设备的标准而开发的，但它可以用来将任何符合要求的设备（真实的或模拟的）与驱动程序连接。

For illustrative purposes, this document will focus on the common case of a Linux kernel running in a virtual machine and using paravirtualized devices provided by the hypervisor, which exposes them as virtio devices via standard mechanisms such as PCI.

为了说明问题，本文将重点讨论Linux内核在虚拟机中运行并使用由管理程序提供的准虚拟化设备的常见情况，管理程序通过标准机制（如PCI）将它们暴露为virtio设备。

Device - Driver communication: virtqueues

Although the virtio devices are really an abstraction layer in the hypervisor, they’re exposed to the guest as if they are physical devices using a specific transport method – PCI, MMIO or CCW – that is orthogonal to the device itself. The virtio spec defines these transport methods in detail, including device discovery, capabilities and interrupt handling.

尽管virtio设备实际上是管理程序中的一个抽象层，但它们被暴露给客户，就像它们是使用特定的传输方法–PCI、MMIO或CCW–的物理设备一样，这与设备本身是正交的。virtio规范详细定义了这些传输方法，包括设备发现、能力和中断处理。

The communication between the driver in the guest OS and the device in the hypervisor is done through shared memory (that’s what makes virtio devices so efficient) using specialized data structures called virtqueues, which are actually ring buffers 1 of buffer descriptors similar to the ones used in a network device:

客户操作系统中的驱动程序和管理程序中的设备之间的通信是通过共享内存完成的（这就是virtio设备如此高效的原因），使用称为virtqueues的专门数据结构，这实际上是类似于网络设备中使用的缓冲区描述符的环形缓冲区1。

struct vring_desc

Virtio ring descriptors, 16 bytes long. These can chain together via next.

Definition:

1
2
3
4
5
6
struct vring_desc {
    __virtio64 addr;
    __virtio32 len;
    __virtio16 flags;
    __virtio16 next;
};

Members

  • addr

    buffer address (guest-physical)

  • len

    buffer length

  • flags

    descriptor flags

  • next

    index of the next descriptor in the chain, if the VRING_DESC_F_NEXT flag is set. We chain unused descriptors via this, too.

All the buffers the descriptors point to are allocated by the guest and used by the host either for reading or for writing but not for both.

Refer to Chapter 2.5 (“Virtqueues”) of the virtio spec ([1]) for the reference definitions of virtqueues and “Virtqueues and virtio ring: How the data travels” blog post ([2]) for an illustrated overview of how the host device and the guest driver communicate.

描述符指向的所有缓冲区都是由guest分配的，并由host用于读取或写入，但不能同时使用。

请参考virtio规范（[1]）的第2.5章（”虚拟队列”），了解虚拟队列的参考定义和 “虚拟队列和virtio环。数据是如何传输的 “博文（[2]），以图文并茂的方式概述了主机设备和客户驱动的通信方式。

The vring_virtqueue struct models a virtqueue, including the ring buffers and management data. Embedded in this struct is the virtqueue struct, which is the data structure that’s ultimately used by virtio drivers:

struct virtqueue

a queue to register buffers for sending or receiving.

Definition:

1
2
3
4
5
6
7
8
9
10
11
struct virtqueue {
    struct list_head list;
    void (*callback)(struct virtqueue *vq);
    const char *name;
    struct virtio_device *vdev;
    unsigned int index;
    unsigned int num_free;
    unsigned int num_max;
    void *priv;
    bool reset;
};

Members

  • list

    the chain of virtqueues for this device

  • callback

    the function to call when buffers are consumed (can be NULL).

  • name

    the name of this virtqueue (mainly for debugging)

  • vdev

    the virtio device this queue was created for.

  • index

    the zero-based ordinal number for this queue.

  • num_free

    number of elements we expect to be able to fit.

  • num_max

    the maximum number of elements supported by the device.

  • priv

    a pointer for the virtqueue implementation to use.

  • reset

    vq is in reset state or not.

Description

A note on num_free: with indirect buffers, each buffer needs one element in the queue, otherwise a buffer will need one element per sg element.

The callback function pointed by this struct is triggered when the device has consumed the buffers provided by the driver. More specifically, the trigger will be an interrupt issued by the hypervisor (see vring_interrupt()). Interrupt request handlers are registered for a virtqueue during the virtqueue setup process (transport-specific).

关于num_free的说明：对于间接缓冲区，每个缓冲区需要队列中的一个元素，否则一个缓冲区将需要每个sg元素的一个元素。

当设备消耗完驱动提供的缓冲区时，这个结构所指向的回调函数会被触发。更具体地说，触发器将是由管理程序发出的中断（见vring_interrupt()）。中断请求处理程序是在虚拟队列设置过程中为虚拟队列注册的（特定于传输）。

irqreturn_t vring_interrupt(int irq, void *_vq)

notify a virtqueue on an interrupt

Parameters

Description

Calls the callback function of _vq to process the virtqueue notification.

Device discovery and probing

In the kernel, the virtio core contains the virtio bus driver and transport-specific drivers like virtio-pci and virtio-mmio. Then there are individual virtio drivers for specific device types that are registered to the virtio bus driver.

在内核中，virtio核心包含virtio总线驱动和特定的传输驱动，如virtio-pci和virtio-mmio。然后，还有针对特定设备类型的单独的virtio驱动程序，它们被注册到virtio总线驱动程序上。

How a virtio device is found and configured by the kernel depends on how the hypervisor defines it. Taking the QEMU virtio-console device as an example. When using PCI as a transport method, the device will present itself on the PCI bus with vendor 0x1af4 (Red Hat, Inc.) and device id 0x1003 (virtio console), as defined in the spec, so the kernel will detect it as it would do with any other PCI device.

内核如何发现和配置virtio设备，取决于管理程序如何定义它。以QEMU virtio-console设备为例。当使用PCI作为传输方式时，该设备将在PCI总线上以供应商0x1af4（Red Hat, Inc.）和设备ID 0x1003（virtio console）的形式出现，正如规范中所定义的那样，所以内核会像检测其他PCI设备那样检测它。

During the PCI enumeration process, if a device is found to match the virtio-pci driver (according to the virtio-pci device table, any PCI device with vendor id = 0x1af4):

在PCI枚举过程中，如果发现一个设备与virtio-pci驱动相匹配（根据virtio-pci设备表，任何PCI设备的厂商ID=0x1af4）。

1
2
3
4
5
/* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
static const struct pci_device_id virtio_pci_id_table[] = {
        { PCI_DEVICE(PCI_VENDOR_ID_REDHAT_QUMRANET, PCI_ANY_ID) },
        { 0 }
};

then the virtio-pci driver is probed and, if the probing goes well, the device is registered to the virtio bus:

然后对virtio-pci驱动进行探测，如果探测顺利，该设备就被注册到virtio总线上。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
static int virtio_pci_probe(struct pci_dev *pci_dev,
                            const struct pci_device_id *id)
{
        ...

        if (force_legacy) {
                rc = virtio_pci_legacy_probe(vp_dev);
                /* Also try modern mode if we can't map BAR0 (no IO space). */
                if (rc == -ENODEV || rc == -ENOMEM)
                        rc = virtio_pci_modern_probe(vp_dev);
                if (rc)
                        goto err_probe;
        } else {
                rc = virtio_pci_modern_probe(vp_dev);
                if (rc == -ENODEV)
                        rc = virtio_pci_legacy_probe(vp_dev);
                if (rc)
                        goto err_probe;
        }

        ...

        rc = register_virtio_device(&vp_dev->vdev);

When the device is registered to the virtio bus the kernel will look for a driver in the bus that can handle the device and call that driver’s probe method.

At this point, the virtqueues will be allocated and configured by calling the appropriate virtio_find helper function, such as virtio_find_single_vq() or virtio_find_vqs(), which will end up calling a transport-specific find_vqs method.

当设备被注册到virtio总线上时，内核将在总线上寻找一个可以处理该设备的驱动程序，并调用该驱动程序的探测方法。

此时，将通过调用适当的virtio_find辅助函数，如virtio_find_single_vq()或virtio_find_vqs()来分配和配置virtqueues，最终会调用一个特定于传输的find_vqs方法。

virtualization►libvirt

Cpu feature configuration code diving

If disable a feature in libvirt domain xml configuration, what will happen?

General code about libvirt cpu conf

Read cpu_conf.c main entrance is virCPUDefFormatBuf

Libvirt have two types format:

  • CUSTOM: user define model and features of a cpu conf
  • HOST_MODEL: matches a most suitable feature list with host

And while handle conf definition:

1
2
3
4
5
formatModel = (def->mode == VIR_CPU_MODE_CUSTOM ||
               def->mode == VIR_CPU_MODE_HOST_MODEL);
formatFallback = (def->type == VIR_CPU_TYPE_GUEST &&
                  (def->mode == VIR_CPU_MODE_HOST_MODEL ||
                   (def->mode == VIR_CPU_MODE_CUSTOM && def->model)));

see the enum:

1
2
3
4
5
6
7
typedef enum {
    VIR_CPU_TYPE_HOST,
    VIR_CPU_TYPE_GUEST,
    VIR_CPU_TYPE_AUTO,

    VIR_CPU_TYPE_LAST
} virCPUType;
  • VIR_CPU_TYPE_AUTO : detect the input xml to tell is guest or host cpu model definition
  • VIR_CPU_TYPE_GUEST : guest cpu model means the cpu conf define from domain xml
  • VIR_CPU_TYPE_HOST : host cpu model means the cpu conf load from host capabilities xml

So the could focus on formatFallback.

Verification is required, if you use a custom mode without a cpu model is not allowed, because custom means you need specify a collections of cpu features and custom features of the subset.

1
2
3
4
5
if (!def->model && def->mode == VIR_CPU_MODE_CUSTOM && def->nfeatures) {
    virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                   _("Non-empty feature list specified without CPU model"));
    return -1;
}

while define model, need to get a fallback value for guest cpu

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
if ((formatModel && def->model) || formatFallback) {
    virBufferAddLit(buf, "<model");
    if (formatFallback) {
        const char *fallback;

        fallback = virCPUFallbackTypeToString(def->fallback);
        if (!fallback) {
            virReportError(VIR_ERR_INTERNAL_ERROR,
                           _("Unexpected CPU fallback value: %d"),
                           def->fallback);
            return -1;
        }
        virBufferAsprintf(buf, " fallback='%s'", fallback);
        if (def->vendor_id)
            virBufferEscapeString(buf, " vendor_id='%s'", def->vendor_id);
    }
    if (formatModel && def->model) {
        virBufferEscapeString(buf, ">%s</model>\n", def->model);
    } else {
        virBufferAddLit(buf, "/>\n");
    }
}

Fallback type:

1
2
3
4
5
6
typedef enum {
    VIR_CPU_FALLBACK_ALLOW,
    VIR_CPU_FALLBACK_FORBID,

    VIR_CPU_FALLBACK_LAST
} virCPUFallback;
  • VIR_CPU_FALLBACK_ALLOW means just use the cpu capabilities from host capabilities xml
  • VIR_CPU_FALLBACK_FORBIDmeans can stop guest from start with unsupported feature

Also the topology can be defined:

1
2
3
4
5
6
7
if (def->sockets && def->cores && def->threads) {
    virBufferAddLit(buf, "<topology");
    virBufferAsprintf(buf, " sockets='%u'", def->sockets);
    virBufferAsprintf(buf, " cores='%u'", def->cores);
    virBufferAsprintf(buf, " threads='%u'", def->threads);
    virBufferAddLit(buf, "/>\n");
}

from xml too.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
for (i = 0; i < def->nfeatures; i++) {
    virCPUFeatureDefPtr feature = def->features + i;

    if (!feature->name) {
        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                       _("Missing CPU feature name"));
        return -1;
    }

    if (def->type == VIR_CPU_TYPE_GUEST) {
        const char *policy;

        policy = virCPUFeaturePolicyTypeToString(feature->policy);
        if (!policy) {
            virReportError(VIR_ERR_INTERNAL_ERROR,
                           _("Unexpected CPU feature policy %d"),
                           feature->policy);
            return -1;
        }
        virBufferAsprintf(buf, "<feature policy='%s' name='%s'/>\n",
                          policy, feature->name);
    } else {
        virBufferAsprintf(buf, "<feature name='%s'/>\n",
                          feature->name);
    }
}

Features will follow policies:

1
2
3
4
5
6
VIR_ENUM_IMPL(virCPUFeaturePolicy, VIR_CPU_FEATURE_LAST,
              "force",
              "require",
              "optional",
              "disable",
              "forbid")

Following part explains about those policies.

force

The virtual CPU will claim the feature is supported regardless of it being supported by host CPU.

require

Guest creation will fail unless the feature is supported by the host CPU or the hypervisor is able to emulate it.

optional

The feature will be supported by virtual CPU if and only if it is supported by host CPU.

disable

The feature will not be supported by virtual CPU.

forbid

Guest creation will fail if the feature is supported by host CPU.

virCPUDefFormatBuf is used by capabilities.c which collects host features from host capabilities xml. But now we need to check the code in domain_capabilities.c

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
typedef virCPUDef *virCPUDefPtr;
struct _virCPUDef {
    int type;           /* enum virCPUType */
    int mode;           /* enum virCPUMode */
    int match;          /* enum virCPUMatch */
    virCPUCheck check;
    virArch arch;
    char *model;
    char *vendor_id;    /* vendor id returned by CPUID in the guest */
    int fallback;       /* enum virCPUFallback */
    char *vendor;
    unsigned int microcodeVersion;
    unsigned int sockets;
    unsigned int cores;
    unsigned int threads;
    size_t nfeatures;
    size_t nfeatures_max;
    virCPUFeatureDefPtr features;
    virCPUCacheDefPtr cache;
};

nfeatures will be set in _virCPUDef and supported features are parsed from domain xml:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/*
 * Parses CPU definition XML from a node pointed to by @xpath. If @xpath is
 * NULL, the current node of @ctxt is used (i.e., it is a shortcut to ".").
 *
 * Missing <cpu> element in the XML document is not considered an error unless
 * @xpath is NULL in which case the function expects it was provided with a
 * valid <cpu> element already. In other words, the function returns success
 * and sets @cpu to NULL if @xpath is not NULL and the node pointed to by
 * @xpath is not found.
 *
 * Returns 0 on success, -1 on error.
 */
int
virCPUDefParseXML(xmlXPathContextPtr ctxt,
                  const char *xpath,
                  virCPUType type,
                  virCPUDefPtr *cpu)

Finally, qemu_command.c would use those features to qemu commandline:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
for (i = 0; i < cpu->nfeatures; i++) {
    if (STREQ("rtm", cpu->features[i].name))
        rtm = true;
    if (STREQ("hle", cpu->features[i].name))
        hle = true;

    switch ((virCPUFeaturePolicy) cpu->features[i].policy) {
    case VIR_CPU_FEATURE_FORCE:
    case VIR_CPU_FEATURE_REQUIRE:
        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_QUERY_CPU_MODEL_EXPANSION))
            virBufferAsprintf(buf, ",%s=on", cpu->features[i].name);
        else
            virBufferAsprintf(buf, ",+%s", cpu->features[i].name);
        break;

    case VIR_CPU_FEATURE_DISABLE:
    case VIR_CPU_FEATURE_FORBID:
        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_QUERY_CPU_MODEL_EXPANSION))
            virBufferAsprintf(buf, ",%s=off", cpu->features[i].name);
        else
            virBufferAsprintf(buf, ",-%s", cpu->features[i].name);
        break;

    case VIR_CPU_FEATURE_OPTIONAL:
    case VIR_CPU_FEATURE_LAST:
        break;
    }
}

like -cpu ... feature1=on,feature2=off to make those features take effects.

Turn to qemu

Firstly, qemu will parse input -cpu .... string:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
const char *parse_cpu_model(const char *cpu_model)
{
    ObjectClass *oc;
    CPUClass *cc;
    gchar **model_pieces;
    const char *cpu_type;

    model_pieces = g_strsplit(cpu_model, ",", 2);

    oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);
    if (oc == NULL) {
        error_report("unable to find CPU model '%s'", model_pieces[0]);
        g_strfreev(model_pieces);
        exit(EXIT_FAILURE);
    }

    cpu_type = object_class_get_name(oc);
    cc = CPU_CLASS(oc);
    cc->parse_features(cpu_type, model_pieces[1], &error_fatal);
    g_strfreev(model_pieces);
    return cpu_type;
}

An object from oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);will return a cpu object class which support parse features. See following code:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
static void cpu_common_parse_features(const char *typename, char *features,
                                      Error **errp)
{
    char *val;
    static bool cpu_globals_initialized;
    /* Single "key=value" string being parsed */
    char *featurestr = features ? strtok(features, ",") : NULL;

    /* should be called only once, catch invalid users */
    assert(!cpu_globals_initialized);
    cpu_globals_initialized = true;

    while (featurestr) {
        val = strchr(featurestr, '=');
        if (val) {
            GlobalProperty *prop = g_new0(typeof(*prop), 1);
            *val = 0;
            val++;
            prop->driver = typename;
            prop->property = g_strdup(featurestr);
            prop->value = g_strdup(val);
            prop->errp = &error_fatal;
            qdev_prop_register_global(prop);
        } else {
            error_setg(errp, "Expected key=value format, found %s.",
                       featurestr);
            return;
        }
        featurestr = strtok(NULL, ",");
    }
}

key=value format will be parse and store into qemu’s global property.

From: target/i386/cpu.c

qemu defined #define CPUID_EXT_HYPERVISOR (1U << 31) for CPUID EXT to expose hypervisor information.

Then x86 cpu will use those global properties to initialize vcpu:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
/* Parse "+feature,-feature,feature=foo" CPU feature string
 */
static void x86_cpu_parse_featurestr(const char *typename, char *features,
                                     Error **errp)
{
    char *featurestr; /* Single 'key=value" string being parsed */
    static bool cpu_globals_initialized;
    bool ambiguous = false;

    if (cpu_globals_initialized) {
        return;
    }
    cpu_globals_initialized = true;

    if (!features) {
        return;
    }

    for (featurestr = strtok(features, ",");
         featurestr;
         featurestr = strtok(NULL, ",")) {
        const char *name;
        const char *val = NULL;
        char *eq = NULL;
        char num[32];
        GlobalProperty *prop;

        /* Compatibility syntax: */
        if (featurestr[0] == '+') {
            plus_features = g_list_append(plus_features,
                                          g_strdup(featurestr + 1));
            continue;
        } else if (featurestr[0] == '-') {
            minus_features = g_list_append(minus_features,
                                           g_strdup(featurestr + 1));
            continue;
        }

        eq = strchr(featurestr, '=');
        if (eq) {
            *eq++ = 0;
            val = eq;
        } else {
            val = "on";
        }

        feat2prop(featurestr);
        name = featurestr;

        if (g_list_find_custom(plus_features, name, compare_string)) {
            warn_report("Ambiguous CPU model string. "
                        "Don't mix both \"+%s\" and \"%s=%s\"",
                        name, name, val);
            ambiguous = true;
        }
        if (g_list_find_custom(minus_features, name, compare_string)) {
            warn_report("Ambiguous CPU model string. "
                        "Don't mix both \"-%s\" and \"%s=%s\"",
                        name, name, val);
            ambiguous = true;
        }

        /* Special case: */
        if (!strcmp(name, "tsc-freq")) {
            int ret;
            uint64_t tsc_freq;

            ret = qemu_strtosz_metric(val, NULL, &tsc_freq);
            if (ret < 0 || tsc_freq > INT64_MAX) {
                error_setg(errp, "bad numerical value %s", val);
                return;
            }
            snprintf(num, sizeof(num), "%" PRId64, tsc_freq);
            val = num;
            name = "tsc-frequency";
        }

        prop = g_new0(typeof(*prop), 1);
        prop->driver = typename;
        prop->property = g_strdup(name);
        prop->value = g_strdup(val);
        prop->errp = &error_fatal;
        qdev_prop_register_global(prop);
    }

    if (ambiguous) {
        warn_report("Compatibility of ambiguous CPU model "
                    "strings won't be kept on future QEMU versions");
    }
}

which is registered as cc->parse_features = x86_cpu_parse_featurestr;.

features from qemu commandline will be put as global property for x86 cpu.

And before start virtual machine, qemu will insure there is not unavailable or missing features:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
static void x86_cpu_get_unavailable_features(Object *obj, Visitor *v,
                                             const char *name, void *opaque,
                                             Error **errp)
{
    X86CPU *xc = X86_CPU(obj);
    strList *result = NULL;

    x86_cpu_list_feature_names(xc->filtered_features, &result);
    visit_type_strList(v, "unavailable-features", &result, errp);
}

/* Check for missing features that may prevent the CPU class from
 * running using the current machine and accelerator.
 */
static void x86_cpu_class_check_missing_features(X86CPUClass *xcc,
                                                 strList **missing_feats)
{
    X86CPU *xc;
    Error *err = NULL;
    strList **next = missing_feats;

    if (xcc->host_cpuid_required && !accel_uses_host_cpuid()) {
        strList *new = g_new0(strList, 1);
        new->value = g_strdup("kvm");
        *missing_feats = new;
        return;
    }

    xc = X86_CPU(object_new(object_class_get_name(OBJECT_CLASS(xcc))));

    x86_cpu_expand_features(xc, &err);
    if (err) {
        /* Errors at x86_cpu_expand_features should never happen,
         * but in case it does, just report the model as not
         * runnable at all using the "type" property.
         */
        strList *new = g_new0(strList, 1);
        new->value = g_strdup("type");
        *next = new;
        next = &new->next;
    }

    x86_cpu_filter_features(xc, false);

    x86_cpu_list_feature_names(xc->filtered_features, next);

    object_unref(OBJECT(xc));
}

while qemu init cpu:

1
static void x86_cpu_realizefn(DeviceState *dev, Error **errp)

features will be set to a CPU object:

1
2
3
if (!kvm_enabled() || !cpu->expose_kvm) {
    env->features[FEAT_KVM] = 0;
}

we could find “hypervisor” related cpu features defined by FEAT_1_ECX:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[FEAT_1_ECX] = {
    .type = CPUID_FEATURE_WORD,
    .feat_names = {
        "pni" /* Intel,AMD sse3 */, "pclmulqdq", "dtes64", "monitor",
        "ds-cpl", "vmx", "smx", "est",
        "tm2", "ssse3", "cid", NULL,
        "fma", "cx16", "xtpr", "pdcm",
        NULL, "pcid", "dca", "sse4.1",
        "sse4.2", "x2apic", "movbe", "popcnt",
        "tsc-deadline", "aes", "xsave", "osxsave",
        "avx", "f16c", "rdrand", "hypervisor",
    },
    .cpuid = { .eax = 1, .reg = R_ECX, },
    .tcg_features = TCG_EXT_FEATURES,
}

then cpu will read those features with key words:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
/*
 * Finishes initialization of CPUID data, filters CPU feature
 * words based on host availability of each feature.
 *
 * Returns: 0 if all flags are supported by the host, non-zero otherwise.
 */
static void x86_cpu_filter_features(X86CPU *cpu, bool verbose)
{
    CPUX86State *env = &cpu->env;
    FeatureWord w;
    const char *prefix = NULL;

    if (verbose) {
        prefix = accel_uses_host_cpuid()
                 ? "host doesn't support requested feature"
                 : "TCG doesn't support requested feature";
    }

    for (w = 0; w < FEATURE_WORDS; w++) {
        uint64_t host_feat =
            x86_cpu_get_supported_feature_word(w, false);
        uint64_t requested_features = env->features[w];
        uint64_t unavailable_features = requested_features & ~host_feat;
        mark_unavailable_features(cpu, w, unavailable_features, prefix);
    }

    if ((env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_INTEL_PT) &&
        kvm_enabled()) {
        KVMState *s = CPU(cpu)->kvm_state;
        uint32_t eax_0 = kvm_arch_get_supported_cpuid(s, 0x14, 0, R_EAX);
        uint32_t ebx_0 = kvm_arch_get_supported_cpuid(s, 0x14, 0, R_EBX);
        uint32_t ecx_0 = kvm_arch_get_supported_cpuid(s, 0x14, 0, R_ECX);
        uint32_t eax_1 = kvm_arch_get_supported_cpuid(s, 0x14, 1, R_EAX);
        uint32_t ebx_1 = kvm_arch_get_supported_cpuid(s, 0x14, 1, R_EBX);

        if (!eax_0 ||
           ((ebx_0 & INTEL_PT_MINIMAL_EBX) != INTEL_PT_MINIMAL_EBX) ||
           ((ecx_0 & INTEL_PT_MINIMAL_ECX) != INTEL_PT_MINIMAL_ECX) ||
           ((eax_1 & INTEL_PT_MTC_BITMAP) != INTEL_PT_MTC_BITMAP) ||
           ((eax_1 & INTEL_PT_ADDR_RANGES_NUM_MASK) <
                                           INTEL_PT_ADDR_RANGES_NUM) ||
           ((ebx_1 & (INTEL_PT_PSB_BITMAP | INTEL_PT_CYCLE_BITMAP)) !=
                (INTEL_PT_PSB_BITMAP | INTEL_PT_CYCLE_BITMAP)) ||
           (ecx_0 & INTEL_PT_IP_LIP)) {
            /*
             * Processor Trace capabilities aren't configurable, so if the
             * host can't emulate the capabilities we report on
             * cpu_x86_cpuid(), intel-pt can't be enabled on the current host.
             */
            mark_unavailable_features(cpu, FEAT_7_0_EBX, CPUID_7_0_EBX_INTEL_PT, prefix);
        }
    }
}

Mainly the features is set by:

1
2
3
4
5
6
7
for (w = 0; w < FEATURE_WORDS; w++) {
    uint64_t host_feat =
        x86_cpu_get_supported_feature_word(w, false);
    uint64_t requested_features = env->features[w];
    uint64_t unavailable_features = requested_features & ~host_feat;
    mark_unavailable_features(cpu, w, unavailable_features, prefix);
}

this part and supported feature keeps 0 because

requested_features & ~host_feat host unavialable features would be ~ at first.

We can dump those configurations from qemu vcpu to check is usage.

How kernel use it

Then we move to linux kernel check about those features usages.

#define X86_FEATURE_HYPERVISOR (4*32+31) /* Running on a hypervisor */

kernel use X86_FEATURE_HYPERVISOR means if running on hypervisor.

Hand on test

Now try to run a guest detecting hypervisor and figure out how to bypass the detection by virtualization level configs.

Linux

http://www.etallen.com/cpuid.html use a cpuid tool to dump cpu id of a guest to check our configuration.

By run cpuid to dump features, we can see following output with our expected values:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
feature information (1/ecx):
   PNI/SSE3: Prescott New Instructions     = true
   PCLMULDQ instruction                    = true
   DTES64: 64-bit debug store              = false
   MONITOR/MWAIT                           = false
   CPL-qualified debug store               = false
   VMX: virtual machine extensions         = true
   SMX: safer mode extensions              = false
   Enhanced Intel SpeedStep Technology     = false
   TM2: thermal monitor 2                  = false
   SSSE3 extensions                        = true
   context ID: adaptive or shared L1 data  = false
   SDBG: IA32_DEBUG_INTERFACE              = false
   FMA instruction                         = true
   CMPXCHG16B instruction                  = true
   xTPR disable                            = false
   PDCM: perfmon and debug                 = false
   PCID: process context identifiers       = true
   DCA: direct cache access                = false
   SSE4.1 extensions                       = true
   SSE4.2 extensions                       = true
   x2APIC: extended xAPIC support          = true
   MOVBE instruction                       = true
   POPCNT instruction                      = true
   time stamp counter deadline             = true
   AES instruction                         = true
   XSAVE/XSTOR states                      = true
   OS-enabled XSAVE/XSTOR                  = true
   AVX: advanced vector extensions         = true
   F16C half-precision convert instruction = true
   RDRAND instruction                      = true
   hypervisor guest status                 = true

the hypervisor guest status = true matches with linux kernel’s definition.

While with hypervisor feature disabled the output changed to:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
feature information (1/edx):
   x87 FPU on chip                        = true
   VME: virtual-8086 mode enhancement     = true
   DE: debugging extensions               = true
   PSE: page size extensions              = true
   TSC: time stamp counter                = true
   RDMSR and WRMSR support                = true
   PAE: physical address extensions       = true
   MCE: machine check exception           = true
   CMPXCHG8B inst.                        = true
   APIC on chip                           = true
   SYSENTER and SYSEXIT                   = true
   MTRR: memory type range registers      = true
   PTE global bit                         = true
   MCA: machine check architecture        = true
   CMOV: conditional move/compare instr   = true
   PAT: page attribute table              = true
   PSE-36: page size extension            = true
   PSN: processor serial number           = false
   CLFLUSH instruction                    = true
   DS: debug store                        = false
   ACPI: thermal monitor and clock ctrl   = false
   MMX Technology                         = true
   FXSAVE/FXRSTOR                         = true
   SSE extensions                         = true
   SSE2 extensions                        = true
   SS: self snoop                         = true
   hyper-threading / multi-core supported = true
   TM: therm. monitor                     = false
   IA64                                   = false
   PBE: pending break event               = false
feature information (1/ecx):
   PNI/SSE3: Prescott New Instructions     = true
   PCLMULDQ instruction                    = true
   DTES64: 64-bit debug store              = false
   MONITOR/MWAIT                           = false
   CPL-qualified debug store               = false
   VMX: virtual machine extensions         = true
   SMX: safer mode extensions              = false
   Enhanced Intel SpeedStep Technology     = false
   TM2: thermal monitor 2                  = false
   SSSE3 extensions                        = true
   context ID: adaptive or shared L1 data  = false
   SDBG: IA32_DEBUG_INTERFACE              = false
   FMA instruction                         = true
   CMPXCHG16B instruction                  = true
   xTPR disable                            = false
   PDCM: perfmon and debug                 = false
   PCID: process context identifiers       = true
   DCA: direct cache access                = false
   SSE4.1 extensions                       = true
   SSE4.2 extensions                       = true
   x2APIC: extended xAPIC support          = true
   MOVBE instruction                       = true
   POPCNT instruction                      = true
   time stamp counter deadline             = true
   AES instruction                         = true
   XSAVE/XSTOR states                      = true
   OS-enabled XSAVE/XSTOR                  = true
   AVX: advanced vector extensions         = true
   F16C half-precision convert instruction = true
   RDRAND instruction                      = true
   hypervisor guest status                 = false

the hypervisor guest status = false value changed as expected.

Linux drawbacks

Read the usage about X86_FEATURE_HYPERVISOR in linux kernel. Some drawbacks can be found in kernel code directly.

From qspintlock.h : 

1
2
3
4
5
6
7
8
9
10
11
12
13
/*
 * RHEL7 specific:
 * To provide backward compatibility with pre-7.4 kernel modules that
 * inlines the ticket spinlock unlock code. The virt_spin_lock() function
 * will have to recognize both a lock value of 0 or _Q_UNLOCKED_VAL as
 * being in an unlocked state.
 */
static inline bool virt_spin_lock(struct qspinlock *lock)
{
	int lockval;

	if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
		return false;

Slow spin lock will not be detected.

From paravirt-spinlocks.c:

1
2
3
4
5
6
7
8
static int __init queued_enable_pv_ticketlock(void)
{
	if (!static_cpu_has(X86_FEATURE_HYPERVISOR) ||
	   (pv_lock_ops.queued_spin_lock_slowpath !=
	    native_queued_spin_lock_slowpath))
		static_key_slow_inc(&paravirt_ticketlocks_enabled);
	return 0;
}

From tsc.c:

1
2
3
4
5
6
7
8
9
/*
 * Don't enable ART in a VM, non-stop TSC required,
 * and the TSC counter resets must not occur asynchronously.
 */
if (boot_cpu_has(X86_FEATURE_HYPERVISOR) ||
    !boot_cpu_has(X86_FEATURE_NONSTOP_TSC) ||
    art_to_tsc_denominator < ART_MIN_DENOMINATOR ||
    tsc_async_resets)
	return;

Always run timer will be started which actually should not be enabled.

From apic.c:

1
2
3
if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER) ||
    boot_cpu_has(X86_FEATURE_HYPERVISOR))
	return;

From mshyperv.c:

1
2
if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
	return 0;

Can not detect if run on hyperv.

From radeon_device :

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * radeon_device_is_virtual - check if we are running is a virtual environment
 *
 * Check if the asic has been passed through to a VM (all asics).
 * Used at driver startup.
 * Returns true if virtual or false if not.
 */
bool radeon_device_is_virtual(void)
{
#ifdef CONFIG_X86
	return boot_cpu_has(X86_FEATURE_HYPERVISOR);
#else
	return false;
#endif
}

Radeon gpu will not detect it is running as guest.

For kernel it may failed to detect that it is running over hypervisor. So related performance improvement changed won’t be applied so there will be a performance drop for those guests.

So does the userspace application also can not do specific things without knowing it is running in virtual machine.