virtualization►libvirt

Cpu feature configuration code diving

If disable a feature in libvirt domain xml configuration, what will happen?

General code about libvirt cpu conf

Read cpu_conf.c main entrance is virCPUDefFormatBuf

Libvirt have two types format:

  • CUSTOM: user define model and features of a cpu conf
  • HOST_MODEL: matches a most suitable feature list with host

And while handle conf definition:

1
2
3
4
5
formatModel = (def->mode == VIR_CPU_MODE_CUSTOM ||
               def->mode == VIR_CPU_MODE_HOST_MODEL);
formatFallback = (def->type == VIR_CPU_TYPE_GUEST &&
                  (def->mode == VIR_CPU_MODE_HOST_MODEL ||
                   (def->mode == VIR_CPU_MODE_CUSTOM && def->model)));

see the enum:

1
2
3
4
5
6
7
typedef enum {
    VIR_CPU_TYPE_HOST,
    VIR_CPU_TYPE_GUEST,
    VIR_CPU_TYPE_AUTO,

    VIR_CPU_TYPE_LAST
} virCPUType;
  • VIR_CPU_TYPE_AUTO : detect the input xml to tell is guest or host cpu model definition
  • VIR_CPU_TYPE_GUEST : guest cpu model means the cpu conf define from domain xml
  • VIR_CPU_TYPE_HOST : host cpu model means the cpu conf load from host capabilities xml

So the could focus on formatFallback.

Verification is required, if you use a custom mode without a cpu model is not allowed, because custom means you need specify a collections of cpu features and custom features of the subset.

1
2
3
4
5
if (!def->model && def->mode == VIR_CPU_MODE_CUSTOM && def->nfeatures) {
    virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                   _("Non-empty feature list specified without CPU model"));
    return -1;
}

while define model, need to get a fallback value for guest cpu

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
if ((formatModel && def->model) || formatFallback) {
    virBufferAddLit(buf, "<model");
    if (formatFallback) {
        const char *fallback;

        fallback = virCPUFallbackTypeToString(def->fallback);
        if (!fallback) {
            virReportError(VIR_ERR_INTERNAL_ERROR,
                           _("Unexpected CPU fallback value: %d"),
                           def->fallback);
            return -1;
        }
        virBufferAsprintf(buf, " fallback='%s'", fallback);
        if (def->vendor_id)
            virBufferEscapeString(buf, " vendor_id='%s'", def->vendor_id);
    }
    if (formatModel && def->model) {
        virBufferEscapeString(buf, ">%s</model>\n", def->model);
    } else {
        virBufferAddLit(buf, "/>\n");
    }
}

Fallback type:

1
2
3
4
5
6
typedef enum {
    VIR_CPU_FALLBACK_ALLOW,
    VIR_CPU_FALLBACK_FORBID,

    VIR_CPU_FALLBACK_LAST
} virCPUFallback;
  • VIR_CPU_FALLBACK_ALLOW means just use the cpu capabilities from host capabilities xml
  • VIR_CPU_FALLBACK_FORBIDmeans can stop guest from start with unsupported feature

Also the topology can be defined:

1
2
3
4
5
6
7
if (def->sockets && def->cores && def->threads) {
    virBufferAddLit(buf, "<topology");
    virBufferAsprintf(buf, " sockets='%u'", def->sockets);
    virBufferAsprintf(buf, " cores='%u'", def->cores);
    virBufferAsprintf(buf, " threads='%u'", def->threads);
    virBufferAddLit(buf, "/>\n");
}

from xml too.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
for (i = 0; i < def->nfeatures; i++) {
    virCPUFeatureDefPtr feature = def->features + i;

    if (!feature->name) {
        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                       _("Missing CPU feature name"));
        return -1;
    }

    if (def->type == VIR_CPU_TYPE_GUEST) {
        const char *policy;

        policy = virCPUFeaturePolicyTypeToString(feature->policy);
        if (!policy) {
            virReportError(VIR_ERR_INTERNAL_ERROR,
                           _("Unexpected CPU feature policy %d"),
                           feature->policy);
            return -1;
        }
        virBufferAsprintf(buf, "<feature policy='%s' name='%s'/>\n",
                          policy, feature->name);
    } else {
        virBufferAsprintf(buf, "<feature name='%s'/>\n",
                          feature->name);
    }
}

Features will follow policies:

1
2
3
4
5
6
VIR_ENUM_IMPL(virCPUFeaturePolicy, VIR_CPU_FEATURE_LAST,
              "force",
              "require",
              "optional",
              "disable",
              "forbid")

Following part explains about those policies.

force

The virtual CPU will claim the feature is supported regardless of it being supported by host CPU.

require

Guest creation will fail unless the feature is supported by the host CPU or the hypervisor is able to emulate it.

optional

The feature will be supported by virtual CPU if and only if it is supported by host CPU.

disable

The feature will not be supported by virtual CPU.

forbid

Guest creation will fail if the feature is supported by host CPU.

virCPUDefFormatBuf is used by capabilities.c which collects host features from host capabilities xml. But now we need to check the code in domain_capabilities.c

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
typedef virCPUDef *virCPUDefPtr;
struct _virCPUDef {
    int type;           /* enum virCPUType */
    int mode;           /* enum virCPUMode */
    int match;          /* enum virCPUMatch */
    virCPUCheck check;
    virArch arch;
    char *model;
    char *vendor_id;    /* vendor id returned by CPUID in the guest */
    int fallback;       /* enum virCPUFallback */
    char *vendor;
    unsigned int microcodeVersion;
    unsigned int sockets;
    unsigned int cores;
    unsigned int threads;
    size_t nfeatures;
    size_t nfeatures_max;
    virCPUFeatureDefPtr features;
    virCPUCacheDefPtr cache;
};

nfeatures will be set in _virCPUDef and supported features are parsed from domain xml:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/*
 * Parses CPU definition XML from a node pointed to by @xpath. If @xpath is
 * NULL, the current node of @ctxt is used (i.e., it is a shortcut to ".").
 *
 * Missing <cpu> element in the XML document is not considered an error unless
 * @xpath is NULL in which case the function expects it was provided with a
 * valid <cpu> element already. In other words, the function returns success
 * and sets @cpu to NULL if @xpath is not NULL and the node pointed to by
 * @xpath is not found.
 *
 * Returns 0 on success, -1 on error.
 */
int
virCPUDefParseXML(xmlXPathContextPtr ctxt,
                  const char *xpath,
                  virCPUType type,
                  virCPUDefPtr *cpu)

Finally, qemu_command.c would use those features to qemu commandline:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
for (i = 0; i < cpu->nfeatures; i++) {
    if (STREQ("rtm", cpu->features[i].name))
        rtm = true;
    if (STREQ("hle", cpu->features[i].name))
        hle = true;

    switch ((virCPUFeaturePolicy) cpu->features[i].policy) {
    case VIR_CPU_FEATURE_FORCE:
    case VIR_CPU_FEATURE_REQUIRE:
        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_QUERY_CPU_MODEL_EXPANSION))
            virBufferAsprintf(buf, ",%s=on", cpu->features[i].name);
        else
            virBufferAsprintf(buf, ",+%s", cpu->features[i].name);
        break;

    case VIR_CPU_FEATURE_DISABLE:
    case VIR_CPU_FEATURE_FORBID:
        if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_QUERY_CPU_MODEL_EXPANSION))
            virBufferAsprintf(buf, ",%s=off", cpu->features[i].name);
        else
            virBufferAsprintf(buf, ",-%s", cpu->features[i].name);
        break;

    case VIR_CPU_FEATURE_OPTIONAL:
    case VIR_CPU_FEATURE_LAST:
        break;
    }
}

like -cpu ... feature1=on,feature2=off to make those features take effects.

Turn to qemu

Firstly, qemu will parse input -cpu .... string:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
const char *parse_cpu_model(const char *cpu_model)
{
    ObjectClass *oc;
    CPUClass *cc;
    gchar **model_pieces;
    const char *cpu_type;

    model_pieces = g_strsplit(cpu_model, ",", 2);

    oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);
    if (oc == NULL) {
        error_report("unable to find CPU model '%s'", model_pieces[0]);
        g_strfreev(model_pieces);
        exit(EXIT_FAILURE);
    }

    cpu_type = object_class_get_name(oc);
    cc = CPU_CLASS(oc);
    cc->parse_features(cpu_type, model_pieces[1], &error_fatal);
    g_strfreev(model_pieces);
    return cpu_type;
}

An object from oc = cpu_class_by_name(CPU_RESOLVING_TYPE, model_pieces[0]);will return a cpu object class which support parse features. See following code:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
static void cpu_common_parse_features(const char *typename, char *features,
                                      Error **errp)
{
    char *val;
    static bool cpu_globals_initialized;
    /* Single "key=value" string being parsed */
    char *featurestr = features ? strtok(features, ",") : NULL;

    /* should be called only once, catch invalid users */
    assert(!cpu_globals_initialized);
    cpu_globals_initialized = true;

    while (featurestr) {
        val = strchr(featurestr, '=');
        if (val) {
            GlobalProperty *prop = g_new0(typeof(*prop), 1);
            *val = 0;
            val++;
            prop->driver = typename;
            prop->property = g_strdup(featurestr);
            prop->value = g_strdup(val);
            prop->errp = &error_fatal;
            qdev_prop_register_global(prop);
        } else {
            error_setg(errp, "Expected key=value format, found %s.",
                       featurestr);
            return;
        }
        featurestr = strtok(NULL, ",");
    }
}

key=value format will be parse and store into qemu’s global property.

From: target/i386/cpu.c

qemu defined #define CPUID_EXT_HYPERVISOR (1U << 31) for CPUID EXT to expose hypervisor information.

Then x86 cpu will use those global properties to initialize vcpu:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
/* Parse "+feature,-feature,feature=foo" CPU feature string
 */
static void x86_cpu_parse_featurestr(const char *typename, char *features,
                                     Error **errp)
{
    char *featurestr; /* Single 'key=value" string being parsed */
    static bool cpu_globals_initialized;
    bool ambiguous = false;

    if (cpu_globals_initialized) {
        return;
    }
    cpu_globals_initialized = true;

    if (!features) {
        return;
    }

    for (featurestr = strtok(features, ",");
         featurestr;
         featurestr = strtok(NULL, ",")) {
        const char *name;
        const char *val = NULL;
        char *eq = NULL;
        char num[32];
        GlobalProperty *prop;

        /* Compatibility syntax: */
        if (featurestr[0] == '+') {
            plus_features = g_list_append(plus_features,
                                          g_strdup(featurestr + 1));
            continue;
        } else if (featurestr[0] == '-') {
            minus_features = g_list_append(minus_features,
                                           g_strdup(featurestr + 1));
            continue;
        }

        eq = strchr(featurestr, '=');
        if (eq) {
            *eq++ = 0;
            val = eq;
        } else {
            val = "on";
        }

        feat2prop(featurestr);
        name = featurestr;

        if (g_list_find_custom(plus_features, name, compare_string)) {
            warn_report("Ambiguous CPU model string. "
                        "Don't mix both \"+%s\" and \"%s=%s\"",
                        name, name, val);
            ambiguous = true;
        }
        if (g_list_find_custom(minus_features, name, compare_string)) {
            warn_report("Ambiguous CPU model string. "
                        "Don't mix both \"-%s\" and \"%s=%s\"",
                        name, name, val);
            ambiguous = true;
        }

        /* Special case: */
        if (!strcmp(name, "tsc-freq")) {
            int ret;
            uint64_t tsc_freq;

            ret = qemu_strtosz_metric(val, NULL, &tsc_freq);
            if (ret < 0 || tsc_freq > INT64_MAX) {
                error_setg(errp, "bad numerical value %s", val);
                return;
            }
            snprintf(num, sizeof(num), "%" PRId64, tsc_freq);
            val = num;
            name = "tsc-frequency";
        }

        prop = g_new0(typeof(*prop), 1);
        prop->driver = typename;
        prop->property = g_strdup(name);
        prop->value = g_strdup(val);
        prop->errp = &error_fatal;
        qdev_prop_register_global(prop);
    }

    if (ambiguous) {
        warn_report("Compatibility of ambiguous CPU model "
                    "strings won't be kept on future QEMU versions");
    }
}

which is registered as cc->parse_features = x86_cpu_parse_featurestr;.

features from qemu commandline will be put as global property for x86 cpu.

And before start virtual machine, qemu will insure there is not unavailable or missing features:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
static void x86_cpu_get_unavailable_features(Object *obj, Visitor *v,
                                             const char *name, void *opaque,
                                             Error **errp)
{
    X86CPU *xc = X86_CPU(obj);
    strList *result = NULL;

    x86_cpu_list_feature_names(xc->filtered_features, &result);
    visit_type_strList(v, "unavailable-features", &result, errp);
}

/* Check for missing features that may prevent the CPU class from
 * running using the current machine and accelerator.
 */
static void x86_cpu_class_check_missing_features(X86CPUClass *xcc,
                                                 strList **missing_feats)
{
    X86CPU *xc;
    Error *err = NULL;
    strList **next = missing_feats;

    if (xcc->host_cpuid_required && !accel_uses_host_cpuid()) {
        strList *new = g_new0(strList, 1);
        new->value = g_strdup("kvm");
        *missing_feats = new;
        return;
    }

    xc = X86_CPU(object_new(object_class_get_name(OBJECT_CLASS(xcc))));

    x86_cpu_expand_features(xc, &err);
    if (err) {
        /* Errors at x86_cpu_expand_features should never happen,
         * but in case it does, just report the model as not
         * runnable at all using the "type" property.
         */
        strList *new = g_new0(strList, 1);
        new->value = g_strdup("type");
        *next = new;
        next = &new->next;
    }

    x86_cpu_filter_features(xc, false);

    x86_cpu_list_feature_names(xc->filtered_features, next);

    object_unref(OBJECT(xc));
}

while qemu init cpu:

1
static void x86_cpu_realizefn(DeviceState *dev, Error **errp)

features will be set to a CPU object:

1
2
3
if (!kvm_enabled() || !cpu->expose_kvm) {
    env->features[FEAT_KVM] = 0;
}

we could find “hypervisor” related cpu features defined by FEAT_1_ECX:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[FEAT_1_ECX] = {
    .type = CPUID_FEATURE_WORD,
    .feat_names = {
        "pni" /* Intel,AMD sse3 */, "pclmulqdq", "dtes64", "monitor",
        "ds-cpl", "vmx", "smx", "est",
        "tm2", "ssse3", "cid", NULL,
        "fma", "cx16", "xtpr", "pdcm",
        NULL, "pcid", "dca", "sse4.1",
        "sse4.2", "x2apic", "movbe", "popcnt",
        "tsc-deadline", "aes", "xsave", "osxsave",
        "avx", "f16c", "rdrand", "hypervisor",
    },
    .cpuid = { .eax = 1, .reg = R_ECX, },
    .tcg_features = TCG_EXT_FEATURES,
}

then cpu will read those features with key words:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
/*
 * Finishes initialization of CPUID data, filters CPU feature
 * words based on host availability of each feature.
 *
 * Returns: 0 if all flags are supported by the host, non-zero otherwise.
 */
static void x86_cpu_filter_features(X86CPU *cpu, bool verbose)
{
    CPUX86State *env = &cpu->env;
    FeatureWord w;
    const char *prefix = NULL;

    if (verbose) {
        prefix = accel_uses_host_cpuid()
                 ? "host doesn't support requested feature"
                 : "TCG doesn't support requested feature";
    }

    for (w = 0; w < FEATURE_WORDS; w++) {
        uint64_t host_feat =
            x86_cpu_get_supported_feature_word(w, false);
        uint64_t requested_features = env->features[w];
        uint64_t unavailable_features = requested_features & ~host_feat;
        mark_unavailable_features(cpu, w, unavailable_features, prefix);
    }

    if ((env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_INTEL_PT) &&
        kvm_enabled()) {
        KVMState *s = CPU(cpu)->kvm_state;
        uint32_t eax_0 = kvm_arch_get_supported_cpuid(s, 0x14, 0, R_EAX);
        uint32_t ebx_0 = kvm_arch_get_supported_cpuid(s, 0x14, 0, R_EBX);
        uint32_t ecx_0 = kvm_arch_get_supported_cpuid(s, 0x14, 0, R_ECX);
        uint32_t eax_1 = kvm_arch_get_supported_cpuid(s, 0x14, 1, R_EAX);
        uint32_t ebx_1 = kvm_arch_get_supported_cpuid(s, 0x14, 1, R_EBX);

        if (!eax_0 ||
           ((ebx_0 & INTEL_PT_MINIMAL_EBX) != INTEL_PT_MINIMAL_EBX) ||
           ((ecx_0 & INTEL_PT_MINIMAL_ECX) != INTEL_PT_MINIMAL_ECX) ||
           ((eax_1 & INTEL_PT_MTC_BITMAP) != INTEL_PT_MTC_BITMAP) ||
           ((eax_1 & INTEL_PT_ADDR_RANGES_NUM_MASK) <
                                           INTEL_PT_ADDR_RANGES_NUM) ||
           ((ebx_1 & (INTEL_PT_PSB_BITMAP | INTEL_PT_CYCLE_BITMAP)) !=
                (INTEL_PT_PSB_BITMAP | INTEL_PT_CYCLE_BITMAP)) ||
           (ecx_0 & INTEL_PT_IP_LIP)) {
            /*
             * Processor Trace capabilities aren't configurable, so if the
             * host can't emulate the capabilities we report on
             * cpu_x86_cpuid(), intel-pt can't be enabled on the current host.
             */
            mark_unavailable_features(cpu, FEAT_7_0_EBX, CPUID_7_0_EBX_INTEL_PT, prefix);
        }
    }
}

Mainly the features is set by:

1
2
3
4
5
6
7
for (w = 0; w < FEATURE_WORDS; w++) {
    uint64_t host_feat =
        x86_cpu_get_supported_feature_word(w, false);
    uint64_t requested_features = env->features[w];
    uint64_t unavailable_features = requested_features & ~host_feat;
    mark_unavailable_features(cpu, w, unavailable_features, prefix);
}

this part and supported feature keeps 0 because

requested_features & ~host_feat host unavialable features would be ~ at first.

We can dump those configurations from qemu vcpu to check is usage.

How kernel use it

Then we move to linux kernel check about those features usages.

#define X86_FEATURE_HYPERVISOR (4*32+31) /* Running on a hypervisor */

kernel use X86_FEATURE_HYPERVISOR means if running on hypervisor.

Hand on test

Now try to run a guest detecting hypervisor and figure out how to bypass the detection by virtualization level configs.

Linux

http://www.etallen.com/cpuid.html use a cpuid tool to dump cpu id of a guest to check our configuration.

By run cpuid to dump features, we can see following output with our expected values:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
feature information (1/ecx):
   PNI/SSE3: Prescott New Instructions     = true
   PCLMULDQ instruction                    = true
   DTES64: 64-bit debug store              = false
   MONITOR/MWAIT                           = false
   CPL-qualified debug store               = false
   VMX: virtual machine extensions         = true
   SMX: safer mode extensions              = false
   Enhanced Intel SpeedStep Technology     = false
   TM2: thermal monitor 2                  = false
   SSSE3 extensions                        = true
   context ID: adaptive or shared L1 data  = false
   SDBG: IA32_DEBUG_INTERFACE              = false
   FMA instruction                         = true
   CMPXCHG16B instruction                  = true
   xTPR disable                            = false
   PDCM: perfmon and debug                 = false
   PCID: process context identifiers       = true
   DCA: direct cache access                = false
   SSE4.1 extensions                       = true
   SSE4.2 extensions                       = true
   x2APIC: extended xAPIC support          = true
   MOVBE instruction                       = true
   POPCNT instruction                      = true
   time stamp counter deadline             = true
   AES instruction                         = true
   XSAVE/XSTOR states                      = true
   OS-enabled XSAVE/XSTOR                  = true
   AVX: advanced vector extensions         = true
   F16C half-precision convert instruction = true
   RDRAND instruction                      = true
   hypervisor guest status                 = true

the hypervisor guest status = true matches with linux kernel’s definition.

While with hypervisor feature disabled the output changed to:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
feature information (1/edx):
   x87 FPU on chip                        = true
   VME: virtual-8086 mode enhancement     = true
   DE: debugging extensions               = true
   PSE: page size extensions              = true
   TSC: time stamp counter                = true
   RDMSR and WRMSR support                = true
   PAE: physical address extensions       = true
   MCE: machine check exception           = true
   CMPXCHG8B inst.                        = true
   APIC on chip                           = true
   SYSENTER and SYSEXIT                   = true
   MTRR: memory type range registers      = true
   PTE global bit                         = true
   MCA: machine check architecture        = true
   CMOV: conditional move/compare instr   = true
   PAT: page attribute table              = true
   PSE-36: page size extension            = true
   PSN: processor serial number           = false
   CLFLUSH instruction                    = true
   DS: debug store                        = false
   ACPI: thermal monitor and clock ctrl   = false
   MMX Technology                         = true
   FXSAVE/FXRSTOR                         = true
   SSE extensions                         = true
   SSE2 extensions                        = true
   SS: self snoop                         = true
   hyper-threading / multi-core supported = true
   TM: therm. monitor                     = false
   IA64                                   = false
   PBE: pending break event               = false
feature information (1/ecx):
   PNI/SSE3: Prescott New Instructions     = true
   PCLMULDQ instruction                    = true
   DTES64: 64-bit debug store              = false
   MONITOR/MWAIT                           = false
   CPL-qualified debug store               = false
   VMX: virtual machine extensions         = true
   SMX: safer mode extensions              = false
   Enhanced Intel SpeedStep Technology     = false
   TM2: thermal monitor 2                  = false
   SSSE3 extensions                        = true
   context ID: adaptive or shared L1 data  = false
   SDBG: IA32_DEBUG_INTERFACE              = false
   FMA instruction                         = true
   CMPXCHG16B instruction                  = true
   xTPR disable                            = false
   PDCM: perfmon and debug                 = false
   PCID: process context identifiers       = true
   DCA: direct cache access                = false
   SSE4.1 extensions                       = true
   SSE4.2 extensions                       = true
   x2APIC: extended xAPIC support          = true
   MOVBE instruction                       = true
   POPCNT instruction                      = true
   time stamp counter deadline             = true
   AES instruction                         = true
   XSAVE/XSTOR states                      = true
   OS-enabled XSAVE/XSTOR                  = true
   AVX: advanced vector extensions         = true
   F16C half-precision convert instruction = true
   RDRAND instruction                      = true
   hypervisor guest status                 = false

the hypervisor guest status = false value changed as expected.

Linux drawbacks

Read the usage about X86_FEATURE_HYPERVISOR in linux kernel. Some drawbacks can be found in kernel code directly.

From qspintlock.h : 

1
2
3
4
5
6
7
8
9
10
11
12
13
/*
 * RHEL7 specific:
 * To provide backward compatibility with pre-7.4 kernel modules that
 * inlines the ticket spinlock unlock code. The virt_spin_lock() function
 * will have to recognize both a lock value of 0 or _Q_UNLOCKED_VAL as
 * being in an unlocked state.
 */
static inline bool virt_spin_lock(struct qspinlock *lock)
{
	int lockval;

	if (!static_cpu_has(X86_FEATURE_HYPERVISOR))
		return false;

Slow spin lock will not be detected.

From paravirt-spinlocks.c:

1
2
3
4
5
6
7
8
static int __init queued_enable_pv_ticketlock(void)
{
	if (!static_cpu_has(X86_FEATURE_HYPERVISOR) ||
	   (pv_lock_ops.queued_spin_lock_slowpath !=
	    native_queued_spin_lock_slowpath))
		static_key_slow_inc(&paravirt_ticketlocks_enabled);
	return 0;
}

From tsc.c:

1
2
3
4
5
6
7
8
9
/*
 * Don't enable ART in a VM, non-stop TSC required,
 * and the TSC counter resets must not occur asynchronously.
 */
if (boot_cpu_has(X86_FEATURE_HYPERVISOR) ||
    !boot_cpu_has(X86_FEATURE_NONSTOP_TSC) ||
    art_to_tsc_denominator < ART_MIN_DENOMINATOR ||
    tsc_async_resets)
	return;

Always run timer will be started which actually should not be enabled.

From apic.c:

1
2
3
if (!boot_cpu_has(X86_FEATURE_TSC_DEADLINE_TIMER) ||
    boot_cpu_has(X86_FEATURE_HYPERVISOR))
	return;

From mshyperv.c:

1
2
if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
	return 0;

Can not detect if run on hyperv.

From radeon_device :

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/**
 * radeon_device_is_virtual - check if we are running is a virtual environment
 *
 * Check if the asic has been passed through to a VM (all asics).
 * Used at driver startup.
 * Returns true if virtual or false if not.
 */
bool radeon_device_is_virtual(void)
{
#ifdef CONFIG_X86
	return boot_cpu_has(X86_FEATURE_HYPERVISOR);
#else
	return false;
#endif
}

Radeon gpu will not detect it is running as guest.

For kernel it may failed to detect that it is running over hypervisor. So related performance improvement changed won’t be applied so there will be a performance drop for those guests.

So does the userspace application also can not do specific things without knowing it is running in virtual machine.