Revert "Makefile: Switch to the Python script for CONFIG checking"

This reverts commit 5b4c6473a341ebbd88463c908b39bb2820e5c43b.

Reason for revert: b:216919905 - breaks chromeos CQ

Original change's description:
> Makefile: Switch to the Python script for CONFIG checking
>
> Now that the Python script seems to do what we need, switch over to
> use that for checking for allowed ad-hoc CONFIGs.
>
> Sadly we need a work-around for the very old Python 3 version used in
> the chroot.
>
> The new script is better at finding Kconfig options, so this allows some
> reductions in the config_allowed.txt file.
>
> Delete the now-unused shell scripts.
>
> BUG=b:181323955
> BRANCH=none
> TEST=python3 util/test_kconfig_check.py
>
> Signed-off-by: Simon Glass <sjg@chromium.org>
> Change-Id: I2dafc9dfe9d9020638f4e49b5c5ee0fc0b10000b
> Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/2923233
> Reviewed-by: Jack Rosenthal <jrosenth@chromium.org>

Bug: b:181323955
Change-Id: I9b5514f9d2df43033cfd95555612e2e89e6ce724
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/3426237
Reviewed-by: Simon Glass <sjg@chromium.org>
Reviewed-by: Jack Rosenthal <jrosenth@chromium.org>
Commit-Queue: Nicolas Norvez <norvez@chromium.org>
Tested-by: Nicolas Norvez <norvez@chromium.org>

5 files changed, 232 insertions, 17 deletions

diff --git a/Makefile.rules b/Makefile.rules
index 6e15b67860..9808fa712d 100644
--- a/Makefile.rules
+++ b/Makefile.rules
@@ -230,9 +230,7 @@ print-boards:
 		$(sort $(boards)))
 
 ifeq ($(ALLOW_CONFIG),)
-cmd_check_allowed = ./util/kconfig_check.py -c ${config} \
-	 -a util/config_allowed.txt -p PLATFORM_EC_ -s zephyr/ \
-	 -I $(abspath ../../../src/third_party/zephyr/main) check
+cmd_check_allowed = ./util/check_allowed.sh ${config} util/config_allowed.txt .
 else
 cmd_check_allowed = true
 endif
diff --git a/util/build_allowed.sh b/util/build_allowed.sh
new file mode 100755
index 0000000000..9aa769a726
--- /dev/null
+++ b/util/build_allowed.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+#
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Taken from U-Boot and modified.
+
+# This grubby little script creates the list of allowed configurations. This
+# file contains all the config options which are allowed to be used outside
+# Kconfig. Please do not add things to the list. Instead, add your new option to
+# Kconfig.
+#
+# Usage:
+#   build_allowed.sh [-u]
+#
+#   -u : Update the existing allowed file
+
+export LC_ALL=C LC_COLLATE=C
+
+# Current list of allowed ad-hoc CONFIGs
+allowed=util/config_allowed.txt
+
+[ "$1" == "-u" ] && update=1
+
+tmp=$(mktemp -d)
+kconfigs="${tmp}/kconfigs"
+
+#
+# Look for the CONFIG options, excluding those in Kconfig and defconfig files.
+#
+git grep CONFIG_ | \
+	grep -E -vi "(Kconfig:|defconfig:|README|\.py|\.pl:)" \
+	| tr ' \t' '\n' \
+	| sed -n 's/^\(CONFIG_[A-Za-z0-9_]*\).*/\1/p' \
+	| sort | uniq >"${tmp}/allowed.tmp1";
+
+# We need a list of the valid Kconfig options to exclude these from the allowed
+# list.
+find . -type f -name "Kconfig*" -exec cat {} \; | sed -n -e \
+	's/^\s*\(config\|menuconfig\) *\([A-Za-z0-9_]*\)$/CONFIG_\2/p' \
+	| sort | uniq > "${kconfigs}"
+
+# Most Kconfigs follow the pattern of CONFIG_PLATFORM_EC_*.  Strip PLATFORM_EC_
+# from the config name to match the cros-ec namespace.
+sed -e 's/^CONFIG_PLATFORM_EC_/CONFIG_/p' "${kconfigs}" | sort | uniq \
+	> "${tmp}/allowed.tmp2"
+
+# Use only the options that are present in the first file but not the second.
+# These comprise new ad-hoc CONFIG options.
+comm -23 "${tmp}/allowed.tmp1" "${tmp}/allowed.tmp2" \
+	| sort | uniq >"${tmp}/allowed.tmp3"
+
+# If ${allowed} already exists, take the intersection of the current
+# list and the new one.  We do not want to increase the allowed options.
+if [ -r "${allowed}" ]; then
+	comm -12 "${tmp}/allowed.tmp3" "${allowed}" > "${tmp}/allowed.tmp4"
+
+	# Find any ad-hoc configs that now have Kconfig options
+	comm -13 "${tmp}/allowed.tmp4" "${allowed}" > "${tmp}/allowed.tmp5"
+	if [ -n "${update}" ]; then
+		echo >&2 "Removing these CONFIG options from the allowed list:"
+		comm -13 "${tmp}/allowed.tmp4" "${allowed}"
+		mv "${tmp}/allowed.tmp4" "${allowed}"
+	elif [ -s "${tmp}/allowed.tmp5" ]; then
+		echo >&2 "The following options are now in Kconfig:"
+		cat >&2 "${tmp}/allowed.tmp5"
+		echo >&2
+		echo >&2 "Please run this to update the list of allowed ad-hoc"
+		echo >&2 "CONFIGs and include this update in your CL:"
+		echo >&2
+		echo -e >&2 "\t./util/build_allowed.sh -u"
+		exit 1
+	fi
+else
+	# If there is no file yet, add one. This allows it to be regenerated
+	# from scratch if needed.
+	mv "${tmp}/allowed.tmp3" "${allowed}"
+fi
+
+rm -rf "${tmp}"
+
+unset LC_ALL LC_COLLATE
diff --git a/util/check_allowed.sh b/util/check_allowed.sh
new file mode 100755
index 0000000000..5212a55a14
--- /dev/null
+++ b/util/check_allowed.sh
@@ -0,0 +1,99 @@
+#!/bin/bash
+#
+# Copyright 2021 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Taken from U-Boot and modified.
+#
+# Check that the .config file provided does not introduce any new ad-hoc CONFIG
+# options
+#
+# Use util/build_allowed.sh to generate the list of current ad-hoc
+# CONFIG options (those which are not in Kconfig).
+
+# Usage
+#    check_allowed.sh <path to .config> <path to allow file> <source dir>
+#
+# For example:
+#   scripts/check_allowed.sh build/volteer/.config config_allowed.txt .
+
+set -e
+set -u
+
+PROG_NAME="${0##*/}"
+
+usage() {
+	echo >&2 "Check that a build does not introduce new ad-hoc CONFIGs"
+	echo >&2 "Usage:"
+	echo -e >&2 "\t${PROG_NAME} <.config file> <allow file> <source dir>"
+	exit 1
+}
+
+die() {
+	echo >&2 "$1"
+	exit 2
+}
+
+[ $# -ge 3 ] || usage
+
+config="$1"
+allow="$2"
+srctree="$3"
+
+tmp=$(mktemp -d)
+
+# Temporary files
+new_configs="${tmp}/configs"
+suspects="${tmp}/suspects"
+kconfigs="${tmp}/kconfigs"
+ok="${tmp}/ok"
+new_adhoc="${tmp}/adhoc"
+
+export LC_ALL=C LC_COLLATE=C
+
+# Get a sorted list of CONFIG options in the .config file
+sed -n 's/^\(CONFIG_[A-Za-z0-9_]*\).*/\1/p' "${config}" | sort | uniq \
+	>"${new_configs}"
+
+# Find any not mentioned in the allowed file
+comm -23 --check-order "${new_configs}" "${allow}" > "${suspects}" || \
+	die "${allow} must be sorted"
+
+# Find all the Kconfig options so far defined
+find "${srctree}" -type f -name "Kconfig*" -exec cat {} \; | sed -n -e \
+	's/^\s*\(config\|menuconfig\) *\([A-Za-z0-9_]*\)$/CONFIG_\2/p' \
+	| sort | uniq > "${kconfigs}"
+
+# Most Kconfigs follow the pattern of CONFIG_PLATFORM_EC_*.  Strip PLATFORM_EC_
+# from the config name to match the cros-ec namespace.
+sed -e 's/^CONFIG_PLATFORM_EC_/CONFIG_/p' "${kconfigs}" | sort | uniq > "${ok}"
+
+# Complain about any new ad-hoc CONFIGs
+comm -23 "${suspects}" "${ok}" >"${new_adhoc}"
+if [ -s "${new_adhoc}" ]; then
+	echo >&2 "Error: The EC is in the process of migrating to Zephyr."
+	echo -e >&2 "\tZephyr uses Kconfig for configuration rather than"
+	echo -e >&2 "\tad-hoc #defines."
+	echo -e >&2 "\tAny new EC CONFIG options must ALSO be added to Zephyr"
+	echo -e >&2 "\tso that new functionality is available in Zephyr also."
+	echo -e >&2 "\tThe following new ad-hoc CONFIG options were detected:"
+	echo >&2
+	cat >&2 "${new_adhoc}"
+	echo >&2
+	echo >&2 "Please add these via Kconfig instead. Find a suitable Kconfig"
+	echo >&2 "file in zephyr/ and add a 'config' or 'menuconfig' option."
+	echo >&2 "Also see details in http://issuetracker.google.com/181253613"
+	echo >&2
+	echo >&2 "To temporarily disable this, use: ALLOW_CONFIG=1 make ..."
+	rm -rf "${tmp}"
+	exit 1
+else
+	# If we are running in a git repo, check if we can remove some things
+	# from the allowed file
+	if git status 2>/dev/null; then
+		./util/build_allowed.sh
+	fi
+fi
+
+rm -rf "${tmp}"
diff --git a/util/config_allowed.txt b/util/config_allowed.txt
index 3488b27653..16ad96d7b5 100644
--- a/util/config_allowed.txt
+++ b/util/config_allowed.txt
@@ -1,3 +1,4 @@
+CONFIG_
 CONFIG_8042_AUX
 CONFIG_ACCELGYRO_BMI160_INT2_OUTPUT
 CONFIG_ACCELGYRO_BMI160_INT_EVENT
@@ -58,8 +59,11 @@ CONFIG_AON_RAM_BASE
 CONFIG_AON_RAM_SIZE
 CONFIG_AP_HANG_DETECT
 CONFIG_AP_WARM_RESET_INTERRUPT
+CONFIG_ARCH_POSIX
+CONFIG_ARM64
 CONFIG_ARMV7M_CACHE
 CONFIG_ASSEMBLY_MULA32
+CONFIG_AUDIO_CODEC
 CONFIG_AUDIO_CODEC_
 CONFIG_AUDIO_CODEC_CAP_WOV_AUDIO_SHM
 CONFIG_AUDIO_CODEC_CAP_WOV_LANG_SHM
@@ -100,6 +104,7 @@ CONFIG_BATTERY_RETRY_NACK
 CONFIG_BATTERY_VENDOR_PARAM
 CONFIG_BATT_HOST_FULL_FACTOR
 CONFIG_BATT_HOST_SHUTDOWN_PERCENTAGE
+CONFIG_BBRAM_INIT_PRIORITY
 CONFIG_BC12_DETECT_DATA_ROLE_TRIGGER
 CONFIG_BC12_DETECT_MAX14637
 CONFIG_BC12_DETECT_PI3USB9281
@@ -213,6 +218,7 @@ CONFIG_CHIP_MEMORY_REGIONS
 CONFIG_CHIP_PANIC_BACKUP
 CONFIG_CHIP_PRE_INIT
 CONFIG_CHIP_UNCACHED_REGION
+CONFIG_CLOCK_CONTROL_EMUL_LOG_LEVEL
 CONFIG_CLOCK_CRYSTAL
 CONFIG_CLOCK_SRC_EXTERNAL
 CONFIG_CMD_
@@ -339,6 +345,8 @@ CONFIG_CONSOLE_HISTORY
 CONFIG_CONSOLE_INPUT_LINE_SIZE
 CONFIG_CONSOLE_UART
 CONFIG_CONSOLE_VERBOSE
+CONFIG_COVERAGE
+CONFIG_CPU_CORTEX_M
 CONFIG_CPU_PROCHOT_ACTIVE_LOW
 CONFIG_CRC8
 CONFIG_CROS_EC_RO_MEM_SIZE
@@ -347,6 +355,7 @@ CONFIG_CTN730
 CONFIG_CTS_TASK_LIST
 CONFIG_CURVE25519
 CONFIG_CUSTOM_FAN_CONTROL
+CONFIG_DAC
 CONFIG_DATA_RAM_SIZE
 CONFIG_DEBUG_BRINGUP
 CONFIG_DEBUG_DISABLE_WRITE_BUFFER
@@ -361,6 +370,7 @@ CONFIG_DELAY_DSW_PWROK_TO_PWRBTN
 CONFIG_DETACHABLE_BASE
 CONFIG_DEVICE_EVENT
 CONFIG_DEVICE_STATE
+CONFIG_DMA
 CONFIG_DMA_DEFAULT_HANDLERS
 CONFIG_DMA_HELP
 CONFIG_DMA_PAGING
@@ -383,7 +393,9 @@ CONFIG_EC_PROTECTED_STORAGE_OFF
 CONFIG_EC_PROTECTED_STORAGE_SIZE
 CONFIG_EC_WRITABLE_STORAGE_OFF
 CONFIG_EC_WRITABLE_STORAGE_SIZE
+CONFIG_EEPROM
 CONFIG_ENABLE_JTAG_SELECTION
+CONFIG_ESPI_LOG_LEVEL
 CONFIG_EVENT_LOG_SIZE
 CONFIG_EXPERIMENTAL_CONSOLE
 CONFIG_EXTENDED_VERSION_INFO
@@ -451,6 +463,8 @@ CONFIG_GESTURE_TAP_SENSOR
 CONFIG_GESTURE_TAP_THRES_MG
 CONFIG_GMR_TABLET_MODE_CUSTOM
 CONFIG_GPIO_GET_EXTENDED
+CONFIG_GPIO_LOG_LEVEL
+CONFIG_GPIO_NCT38XX
 CONFIG_GPIO_POWER_DOWN
 CONFIG_GPOI_EXTENDED
 CONFIG_GYRO_L3GD20H
@@ -505,13 +519,20 @@ CONFIG_HOSYCMD_BATTERY_V2
 CONFIG_HWTIMER_64BIT
 CONFIG_HW_CRC
 CONFIG_HW_SPECIFIC_UDELAY
+CONFIG_I2C_BITBANG
 CONFIG_I2C_BUS_MAY_BE_UNPOWERED
 CONFIG_I2C_CHIP_MAX_TRANSFER_SIZE
 CONFIG_I2C_CONTROLLER
+CONFIG_I2C_EMUL
 CONFIG_I2C_EXTRA_PACKET_SIZE
 CONFIG_I2C_HID_TOUCHPAD
+CONFIG_I2C_ITE_IT8XXX2
+CONFIG_I2C_LOG_LEVEL
+CONFIG_I2C_LOG_LEVEL_DBG
+CONFIG_I2C_MOCK_LOG_LEVEL
 CONFIG_I2C_MULTI_PORT_CONTROLLER
 CONFIG_I2C_NACK_RETRY_COUNT
+CONFIG_I2C_NPCX
 CONFIG_I2C_PERIPHERAL
 CONFIG_I2C_SCL_GATE_ADDR
 CONFIG_I2C_SCL_GATE_ADDR_FLAGS
@@ -520,6 +541,7 @@ CONFIG_I2C_SCL_GATE_PORT
 CONFIG_I2C_UPDATE_IF_CHANGED
 CONFIG_I2C_XFER_BOARD_CALLBACK
 CONFIG_I2C_XFER_LARGE_TRANSFER
+CONFIG_INA219
 CONFIG_INA231
 CONFIG_INA3221
 CONFIG_INDUCTIVE_CHARGING
@@ -555,6 +577,7 @@ CONFIG_ISH_PM_D0I3
 CONFIG_ISH_PM_D3
 CONFIG_ISH_PM_RESET_PREP
 CONFIG_ISH_UART_0
+CONFIG_ISL923X_EMUL_LOG_LEVEL
 CONFIG_ISL9241_SWITCHING_FREQ
 CONFIG_IT83XX_ENABLE_MOUSE_DEVICE
 CONFIG_IT83XX_FLASH_CLOCK_48MHZ
@@ -566,6 +589,10 @@ CONFIG_IT83XX_TUNE_CC_PHY
 CONFIG_IT83XX_VCC_1P8V
 CONFIG_IT83XX_VCC_3P3V
 CONFIG_ITE_FLASH_SUPPORT
+CONFIG_ITE_IT8XXX2_INTC
+CONFIG_KERNEL_INIT_PRIORITY_DEFAULT
+CONFIG_KERNEL_INIT_PRIORITY_DEVICE
+CONFIG_KERNEL_INIT_PRIORITY_OBJECTS
 CONFIG_KEYBOARD_ASSISTANT_KEY
 CONFIG_KEYBOARD_BACKLIGHT
 CONFIG_KEYBOARD_BOOT_KEYS
@@ -607,11 +634,14 @@ CONFIG_LID_ANGLE_TABLET_MODE
 CONFIG_LID_SWITCH_GPIO_LIST
 CONFIG_LIGHTBAR_POWER_RAILS
 CONFIG_LIGHTBAR_TAP_DIM_LAST_SEGMENT
+CONFIG_LIS2DW12_EMUL_LOG_LEVEL
 CONFIG_LLSR_TEST
 CONFIG_LN9310
+CONFIG_LN9310_EMUL_LOG_LEVEL
 CONFIG_LOADER_MEM_OFF
 CONFIG_LOADER_SIZE
 CONFIG_LOADER_STORAGE_OFF
+CONFIG_LOG
 CONFIG_LOG_MAX_LEVEL
 CONFIG_LOW_POWER_IDLE_LIMITED
 CONFIG_LOW_POWER_S0
@@ -689,10 +719,13 @@ CONFIG_PANIC_DRAM_SIZE
 CONFIG_PANIC_STRIP_GPR
 CONFIG_PD_RETRY_COUNT
 CONFIG_PD_USE_DAC_AS_REF
+CONFIG_PECI
 CONFIG_PECI_COMMON
 CONFIG_PECI_TJMAX
 CONFIG_PERIPHERAL_CHARGER
+CONFIG_PINMUX_INIT_PRIORITY
 CONFIG_PLATFORM_EC_
+CONFIG_PM
 CONFIG_PMIC_FW_LONG_PRESS_TIMER
 CONFIG_PMU_HARD_RESET
 CONFIG_POLLING_UART
@@ -716,6 +749,7 @@ CONFIG_PRINTF_LEGACY_LI_FORMAT
 CONFIG_PRINT_IN_INT
 CONFIG_PROGRAM_MEMORY_BASE
 CONFIG_PROGRAM_MEMORY_BASE_LOAD
+CONFIG_PS2
 CONFIG_PSTORE
 CONFIG_PVD
 CONFIG_PWM_INPUT_LFCLK
@@ -797,11 +831,18 @@ CONFIG_SHELL_HELP
 CONFIG_SIMULATED_BUTTON
 CONFIG_SLEEP_TIMEOUT_MS
 CONFIG_SMBUS
+CONFIG_SN5S330_EMUL_LOG_LEVEL
 CONFIG_SOC
+CONFIG_SOC_FAMILY_NPCX
+CONFIG_SOC_FAMILY_RISCV_ITE
+CONFIG_SOC_IT8XXX2
+CONFIG_SOC_POSIX
 CONFIG_SOC_POWER_MANAGEMENT_TRACE
+CONFIG_SOC_SERIES_NPCX7
 CONFIG_SOFTWARE_CLZ
 CONFIG_SOFTWARE_CTZ
 CONFIG_SOMETHING
+CONFIG_SPI
 CONFIG_SPI_ACCEL_PORT
 CONFIG_SPI_CONTROLLER
 CONFIG_SPI_CS_GPIO
@@ -818,8 +859,10 @@ CONFIG_SPI_FLASH_W25Q80
 CONFIG_SPI_FLASH_W25X40
 CONFIG_SPI_FP_PORT
 CONFIG_SPI_HALFDUPLEX
+CONFIG_SPI_NOR
 CONFIG_SPI_NOR_BLOCK_ERASE
 CONFIG_SPI_NOR_DEBUG
+CONFIG_SPI_NOR_INIT_PRIORITY
 CONFIG_SPI_NOR_MAX_MESSAGE_SIZE
 CONFIG_SPI_NOR_MAX_READ_SIZE
 CONFIG_SPI_NOR_MAX_WRITE_SIZE
@@ -863,6 +906,7 @@ CONFIG_SYV682X_HV_ILIM
 CONFIG_TASK_LIST
 CONFIG_TASK_PROFILING
 CONFIG_TASK_RESET_LIST
+CONFIG_TCPCI_EMUL_LOG_LEVEL
 CONFIG_TCPC_I2C_BASE_ADDR
 CONFIG_TCPC_I2C_BASE_ADDR_FLAGS
 CONFIG_TCS_USE_LUX_TABLE
@@ -906,6 +950,7 @@ CONFIG_TOUCHPAD_VIRTUAL_SIZE
 CONFIG_TRICKLE_CHARGING
 CONFIG_TUSB544_EQ_BY_REGISTER
 CONFIG_UART_BAUD_RATE
+CONFIG_UART_CONSOLE
 CONFIG_UART_HOST
 CONFIG_UART_HOST_COMMAND_BAUD_RATE
 CONFIG_UART_HOST_COMMAND_HW
@@ -1069,6 +1114,7 @@ CONFIG_VALUE
 CONFIG_VBOOT_EFS
 CONFIG_WATCHDOG_HELP
 CONFIG_WATCHDOG_MAX_RETRIES
+CONFIG_WDT_ITE_WARNING_LEADING_TIME_MS
 CONFIG_WEBUSB_URL
 CONFIG_WIRELESS
 CONFIG_WIRELESS_CHARGER_P9221_R7
@@ -1082,4 +1128,5 @@ CONFIG_WP_STORAGE_OFF
 CONFIG_WP_STORAGE_SIZE
 CONFIG_X86_64
 CONFIG_ZEPHYR
+CONFIG_ZTEST
 CONFIG_xxx
diff --git a/util/kconfig_check.py b/util/kconfig_check.py
index 30a798b8f4..93f3afd53c 100755
--- a/util/kconfig_check.py
+++ b/util/kconfig_check.py
@@ -70,12 +70,7 @@ a corresponding Kconfig option for Zephyr'''
     parser.add_argument('-s', '--srctree', type=str, default='zephyr/',
                         help='Path to source tree to look for Kconfigs')
 
-    # TODO(sjg@chromium.org): The chroot uses a very old Python. Once it moves
-    # to 3.7 or later we can use this instead:
-    #    subparsers = parser.add_subparsers(dest='cmd', required=True)
-    subparsers = parser.add_subparsers(dest='cmd')
-    subparsers.required = True
-
+    subparsers = parser.add_subparsers(dest='cmd', required=True)
     subparsers.add_parser('check', help='Check for new ad-hoc CONFIGs')
 
     return parser.parse_args(argv)
@@ -274,14 +269,7 @@ class KconfigCheck:
                     current state of the Kconfig options
         """
         configs = self.read_configs(configs_file, use_defines)
-        try:
-            kconfigs = self.scan_kconfigs(srcdir, prefix, search_paths)
-        except kconfiglib.KconfigError:
-            # If we don't actually have access to the full Kconfig then we may
-            # get an error. Fall back to using manual methods.
-            kconfigs = self.scan_kconfigs(srcdir, prefix, search_paths,
-                                          try_kconfiglib=False)
-
+        kconfigs = self.scan_kconfigs(srcdir, prefix, search_paths)
         allowed = self.read_allowed(allowed_file)
         new_adhoc = self.find_new_adhoc(configs, kconfigs, allowed)
         unneeded_adhoc = self.find_unneeded_adhoc(kconfigs, allowed)