Set smp affinity from affinity hint for gvnic queues (#882)

1 files changed, 22 insertions, 3 deletions

diff --git a/packages/google-compute-engine/src/usr/bin/google_set_multiqueue b/packages/google-compute-engine/src/usr/bin/google_set_multiqueue
index 7064a6b..1cc4da0 100755
--- a/packages/google-compute-engine/src/usr/bin/google_set_multiqueue
+++ b/packages/google-compute-engine/src/usr/bin/google_set_multiqueue
@@ -26,6 +26,14 @@
 # configuration allows network traffic to be spread across the CPUs, giving
 # each CPU a dedicated TX and RX network queue, while ensuring that all packets
 # from a single flow are delivered to the same CPU.
+#
+# For a gvnic device, set the IRQ affinities to the per-IRQ affinity hint. 
+# The google virtual ethernet driver maps each queue MSI-X interrupt to a 
+# unique single CPU, which is stored in the affinity_hint for each MSI-X 
+# vector. In older versions of the kernel, irqblanace is expected to copy the 
+# affinity_hint to smp_affinity; however, GCE instances disable irqbalance by 
+# default. This script copies over the affinity_hint to smp_affinity on boot to
+# replicate the behavior of irqbalance.
 
 function is_decimal_int() {
   [ "${1}" -eq "${1}" ] > /dev/null 2>&1
@@ -36,11 +44,11 @@ function set_channels() {
 }
 
 echo "Running $(basename $0)."
-NET_DEVS=/sys/bus/virtio/drivers/virtio_net/virtio*
+VIRTIO_NET_DEVS=/sys/bus/virtio/drivers/virtio_net/virtio*
 
 # Loop through all the virtionet devices and enable multi-queue
 if [ -x "$(command -v ethtool)" ]; then
-  for dev in $NET_DEVS; do
+  for dev in $VIRTIO_NET_DEVS; do
     ETH_DEVS=${dev}/net/*
     for eth_dev in $ETH_DEVS; do
       eth_dev=$(basename "$eth_dev")
@@ -62,7 +70,7 @@ else
   echo "ethtool not found: cannot configure virtionet multiqueue."
 fi
 
-for dev in $NET_DEVS
+for dev in $VIRTIO_NET_DEVS
 do
     dev=$(basename "$dev")
     irq_dir=/proc/irq/*
@@ -104,6 +112,17 @@ do
     done
 done
 
+# Set smp_affinity properly for gvnic queues. '-ntfy-block.' is unique to gve
+# and will not affect virtio queues.
+for i in /proc/irq/*; do
+  if ls ${i}/*-ntfy-block.* 1> /dev/null 2>&1; then
+    if [ -f ${i}/affinity_hint ]; then
+      echo Setting smp_affinity on ${i} to $(cat ${i}/affinity_hint)
+      cp ${i}/affinity_hint ${i}/smp_affinity
+    fi
+  fi
+done
+
 XPS=/sys/class/net/e*/queues/tx*/xps_cpus
 num_cpus=$(nproc)