diff options
| -rw-r--r-- | lib/automake.mk | 2 | ||||
| -rw-r--r-- | lib/cpu.c | 68 | ||||
| -rw-r--r-- | lib/cpu.h | 34 | ||||
| -rw-r--r-- | lib/dpdk-stub.c | 9 | ||||
| -rw-r--r-- | lib/dpdk.c | 52 | ||||
| -rw-r--r-- | lib/dpdk.h | 1 | ||||
| -rw-r--r-- | lib/dpif-netdev-avx512.c | 5 | ||||
| -rw-r--r-- | lib/dpif-netdev-extract-avx512.c | 14 | ||||
| -rw-r--r-- | lib/dpif-netdev-lookup-avx512-gather.c | 8 |
9 files changed, 119 insertions, 74 deletions
diff --git a/lib/automake.mk b/lib/automake.mk index 46f869a33..5224e0856 100644 --- a/lib/automake.mk +++ b/lib/automake.mk @@ -38,6 +38,8 @@ lib_libopenvswitchavx512_la_CFLAGS = \ -fPIC \ $(AM_CFLAGS) lib_libopenvswitchavx512_la_SOURCES = \ + lib/cpu.c \ + lib/cpu.h \ lib/dpif-netdev-lookup-avx512-gather.c \ lib/dpif-netdev-extract-avx512.c \ lib/dpif-netdev-avx512.c diff --git a/lib/cpu.c b/lib/cpu.c new file mode 100644 index 000000000..2df003c51 --- /dev/null +++ b/lib/cpu.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2021, Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <config.h> + +#include "cpu.h" +#include "openvswitch/compiler.h" + +#ifdef __x86_64__ +#include <cpuid.h> +#include <inttypes.h> + +#include "openvswitch/util.h" + +enum x86_reg { + EAX, + EBX, + ECX, + EDX, +}; +#define X86_LEAF_MASK 0x80000000 +#define X86_EXT_FEATURES_LEAF 0x00000007 +static bool x86_has_isa(uint32_t leaf, enum x86_reg reg, uint32_t bit) +{ + uint32_t regs[4]; + + ovs_assert(__get_cpuid_max(leaf & X86_LEAF_MASK, NULL) >= leaf); + + __cpuid_count(leaf, 0, regs[EAX], regs[EBX], regs[ECX], regs[EDX]); + return (regs[reg] & ((uint32_t) 1 << bit)) != 0; +} + +static bool x86_isa[OVS_CPU_ISA_X86_LAST - OVS_CPU_ISA_X86_FIRST + 1]; +#define X86_ISA(leaf, reg, bit, name) \ +OVS_CONSTRUCTOR(cpu_isa_ ## name) { \ + x86_isa[name - OVS_CPU_ISA_X86_FIRST] = x86_has_isa(leaf, reg, bit); \ +} +X86_ISA(X86_EXT_FEATURES_LEAF, EBX, 8, OVS_CPU_ISA_X86_BMI2) +X86_ISA(X86_EXT_FEATURES_LEAF, EBX, 16, OVS_CPU_ISA_X86_AVX512F) +X86_ISA(X86_EXT_FEATURES_LEAF, EBX, 30, OVS_CPU_ISA_X86_AVX512BW) +X86_ISA(X86_EXT_FEATURES_LEAF, ECX, 1, OVS_CPU_ISA_X86_AVX512VBMI) +X86_ISA(X86_EXT_FEATURES_LEAF, ECX, 14, OVS_CPU_ISA_X86_VPOPCNTDQ) +#endif + +bool +cpu_has_isa(enum ovs_cpu_isa isa OVS_UNUSED) +{ +#ifdef __x86_64__ + if (isa >= OVS_CPU_ISA_X86_FIRST && + isa <= OVS_CPU_ISA_X86_LAST) { + return x86_isa[isa - OVS_CPU_ISA_X86_FIRST]; + } +#endif + return false; +} diff --git a/lib/cpu.h b/lib/cpu.h new file mode 100644 index 000000000..92897bb71 --- /dev/null +++ b/lib/cpu.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021, Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CPU_H +#define CPU_H 1 + +#include <stdbool.h> + +enum ovs_cpu_isa { + OVS_CPU_ISA_X86_FIRST, + OVS_CPU_ISA_X86_BMI2 = OVS_CPU_ISA_X86_FIRST, + OVS_CPU_ISA_X86_AVX512F, + OVS_CPU_ISA_X86_AVX512BW, + OVS_CPU_ISA_X86_AVX512VBMI, + OVS_CPU_ISA_X86_VPOPCNTDQ, + OVS_CPU_ISA_X86_LAST = OVS_CPU_ISA_X86_VPOPCNTDQ, +}; + +bool cpu_has_isa(enum ovs_cpu_isa); + +#endif /* CPU_H */ diff --git a/lib/dpdk-stub.c b/lib/dpdk-stub.c index fe24f9abd..c332c217c 100644 --- a/lib/dpdk-stub.c +++ b/lib/dpdk-stub.c @@ -79,15 +79,6 @@ print_dpdk_version(void) { } -bool -dpdk_get_cpu_has_isa(const char *arch OVS_UNUSED, - const char *feature OVS_UNUSED) -{ - VLOG_DBG_ONCE("DPDK not supported in this version of Open vSwitch, " - "cannot use CPU flag based optimizations"); - return false; -} - void dpdk_status(const struct ovsrec_open_vswitch *cfg) { diff --git a/lib/dpdk.c b/lib/dpdk.c index 6cdd69bd2..38f142017 100644 --- a/lib/dpdk.c +++ b/lib/dpdk.c @@ -592,58 +592,6 @@ print_dpdk_version(void) puts(rte_version()); } -/* Avoid calling rte_cpu_get_flag_enabled() excessively, by caching the - * result of the call for each CPU flag in a static variable. To avoid - * allocating large numbers of static variables, use a uint8 as a bitfield. - * Note the macro must only return if the ISA check is done and available. - */ -#define ISA_CHECK_DONE_BIT (1 << 0) -#define ISA_AVAILABLE_BIT (1 << 1) - -#define CHECK_CPU_FEATURE(feature, name_str, RTE_CPUFLAG) \ - do { \ - if (strncmp(feature, name_str, strlen(name_str)) == 0) { \ - static uint8_t isa_check_##RTE_CPUFLAG; \ - int check = isa_check_##RTE_CPUFLAG & ISA_CHECK_DONE_BIT; \ - if (OVS_UNLIKELY(!check)) { \ - int has_isa = rte_cpu_get_flag_enabled(RTE_CPUFLAG); \ - VLOG_DBG("CPU flag %s, available %s\n", \ - name_str, has_isa ? "yes" : "no"); \ - isa_check_##RTE_CPUFLAG = ISA_CHECK_DONE_BIT; \ - if (has_isa) { \ - isa_check_##RTE_CPUFLAG |= ISA_AVAILABLE_BIT; \ - } \ - } \ - if (isa_check_##RTE_CPUFLAG & ISA_AVAILABLE_BIT) { \ - return true; \ - } else { \ - return false; \ - } \ - } \ - } while (0) - -bool -dpdk_get_cpu_has_isa(const char *arch, const char *feature) -{ - /* Ensure Arch is x86_64. */ - if (strncmp(arch, "x86_64", 6) != 0) { - return false; - } - -#if __x86_64__ - /* CPU flags only defined for the architecture that support it. */ - CHECK_CPU_FEATURE(feature, "avx512f", RTE_CPUFLAG_AVX512F); - CHECK_CPU_FEATURE(feature, "avx512bw", RTE_CPUFLAG_AVX512BW); - CHECK_CPU_FEATURE(feature, "avx512vbmi", RTE_CPUFLAG_AVX512VBMI); - CHECK_CPU_FEATURE(feature, "avx512vpopcntdq", RTE_CPUFLAG_AVX512VPOPCNTDQ); - CHECK_CPU_FEATURE(feature, "bmi2", RTE_CPUFLAG_BMI2); -#endif - - VLOG_WARN("Unknown CPU arch,feature: %s,%s. Returning not supported.\n", - arch, feature); - return false; -} - void dpdk_status(const struct ovsrec_open_vswitch *cfg) { diff --git a/lib/dpdk.h b/lib/dpdk.h index 445a51d06..2eb1aedbb 100644 --- a/lib/dpdk.h +++ b/lib/dpdk.h @@ -44,6 +44,5 @@ bool dpdk_per_port_memory(void); bool dpdk_available(void); void print_dpdk_version(void); void dpdk_status(const struct ovsrec_open_vswitch *); -bool dpdk_get_cpu_has_isa(const char *arch, const char *feature); #endif /* dpdk.h */ diff --git a/lib/dpif-netdev-avx512.c b/lib/dpif-netdev-avx512.c index 3980960ba..560f3decb 100644 --- a/lib/dpif-netdev-avx512.c +++ b/lib/dpif-netdev-avx512.c @@ -20,6 +20,7 @@ #include <config.h> +#include "cpu.h" #include "dpif-netdev.h" #include "dpif-netdev-perf.h" #include "dpif-netdev-private.h" @@ -61,8 +62,8 @@ struct dpif_userdata { int32_t dp_netdev_input_outer_avx512_probe(void) { - bool avx512f_available = dpdk_get_cpu_has_isa("x86_64", "avx512f"); - bool bmi2_available = dpdk_get_cpu_has_isa("x86_64", "bmi2"); + bool avx512f_available = cpu_has_isa(OVS_CPU_ISA_X86_AVX512F); + bool bmi2_available = cpu_has_isa(OVS_CPU_ISA_X86_BMI2); if (!avx512f_available || !bmi2_available) { return -ENOTSUP; diff --git a/lib/dpif-netdev-extract-avx512.c b/lib/dpif-netdev-extract-avx512.c index aea6e1567..e060ab14a 100644 --- a/lib/dpif-netdev-extract-avx512.c +++ b/lib/dpif-netdev-extract-avx512.c @@ -42,8 +42,8 @@ #include <stdint.h> #include <string.h> +#include "cpu.h" #include "flow.h" -#include "dpdk.h" #include "dpif-netdev-private-dpcls.h" #include "dpif-netdev-private-extract.h" @@ -631,21 +631,21 @@ DECLARE_MFEX_FUNC(dot1q_ip_tcp, PROFILE_ETH_VLAN_IPV4_TCP) static int32_t avx512_isa_probe(uint32_t needs_vbmi) { - static const char *isa_required[] = { - "avx512f", - "avx512bw", - "bmi2", + static enum ovs_cpu_isa isa_required[] = { + OVS_CPU_ISA_X86_AVX512F, + OVS_CPU_ISA_X86_AVX512BW, + OVS_CPU_ISA_X86_BMI2, }; int32_t ret = 0; for (uint32_t i = 0; i < ARRAY_SIZE(isa_required); i++) { - if (!dpdk_get_cpu_has_isa("x86_64", isa_required[i])) { + if (!cpu_has_isa(isa_required[i])) { ret = -ENOTSUP; } } if (needs_vbmi) { - if (!dpdk_get_cpu_has_isa("x86_64", "avx512vbmi")) { + if (!cpu_has_isa(OVS_CPU_ISA_X86_AVX512VBMI)) { ret = -ENOTSUP; } } diff --git a/lib/dpif-netdev-lookup-avx512-gather.c b/lib/dpif-netdev-lookup-avx512-gather.c index 072831e96..7bc1e9e9a 100644 --- a/lib/dpif-netdev-lookup-avx512-gather.c +++ b/lib/dpif-netdev-lookup-avx512-gather.c @@ -21,7 +21,9 @@ #include "dpif-netdev.h" #include "dpif-netdev-lookup.h" + #include "cmap.h" +#include "cpu.h" #include "flow.h" #include "pvector.h" #include "openvswitch/vlog.h" @@ -398,13 +400,13 @@ dpcls_subtable_avx512_gather_probe(uint32_t u0_bits, uint32_t u1_bits) { dpcls_subtable_lookup_func f = NULL; - int avx512f_available = dpdk_get_cpu_has_isa("x86_64", "avx512f"); - int bmi2_available = dpdk_get_cpu_has_isa("x86_64", "bmi2"); + int avx512f_available = cpu_has_isa(OVS_CPU_ISA_X86_AVX512F); + int bmi2_available = cpu_has_isa(OVS_CPU_ISA_X86_BMI2); if (!avx512f_available || !bmi2_available) { return NULL; } - int use_vpop = dpdk_get_cpu_has_isa("x86_64", "avx512vpopcntdq"); + int use_vpop = cpu_has_isa(OVS_CPU_ISA_X86_VPOPCNTDQ); CHECK_LOOKUP_FUNCTION(9, 4, use_vpop); CHECK_LOOKUP_FUNCTION(9, 1, use_vpop); |