cmake: support -DENABLE_SANITIZERS={OFF,ON}.

This should keep silly tools that pass -DENABLE_SANITIZERS=OFF to CMake
if no sanitizers have been specified, rather than just not defining
ENABLE_SANITIZERS.

This should fix #1171.

1 files changed, 147 insertions, 37 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 270b6593..315007f0 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1303,50 +1303,160 @@ endif()
 # obvious CMake debugging flag reveals, it doesn't realize that if we
 # turn sanitizer stuff on).
 #
+# Note: apparently, some projects have decided that ENABLE_SANITIZERS
+# is a Boolean, with OFF meaning "no sanitizers" and ON meaning "all
+# sanitizers".  Whoever decided that didn't put it up as a common
+# CMake idiom, as far as I can tell; we only discovered this because
+# JetBrains' CLion "helpfully" appears to pass -DENABLE_SANITIZERS=OFF
+# to CMake by default, which causes CMake to fail on libpcap.  Thanks!
+#
+# We thus also allow a setting of OFF to mean "no sanitizers" and ON to
+# mean "all supported sanitizers that we know about and that can all
+# be used together".
+#
 set(SANITIZER_FLAGS "")
-foreach(sanitizer IN LISTS ENABLE_SANITIZERS)
-    # Set -Werror to catch "argument unused during compilation" warnings
-
-    message(STATUS "Checking sanitizer ${sanitizer}")
-    set(sanitizer_variable "sanitize_${sanitizer}")
-    set(CMAKE_REQUIRED_FLAGS "-Werror -fsanitize=${sanitizer}")
-    check_c_compiler_flag("-fsanitize=${sanitizer}" ${sanitizer_variable})
-    if(${${sanitizer_variable}})
-        set(SANITIZER_FLAGS "${SANITIZER_FLAGS} -fsanitize=${sanitizer}")
-        message(STATUS "${sanitizer} sanitizer supported using -fsanitizer=${sanitizer}")
-    else()
-        #
-        # Try the versions supported prior to Clang 3.2.
-        # If the sanitizer is "address", try -fsanitize-address.
-        # If it's "undefined", try -fcatch-undefined-behavior.
-        # Otherwise, give up.
-        #
-        set(sanitizer_variable "OLD_${sanitizer_variable}")
-        if ("${sanitizer}" STREQUAL "address")
-            set(CMAKE_REQUIRED_FLAGS "-Werror -fsanitize-address")
-            check_c_compiler_flag("-fsanitize-address" ${sanitizer_variable})
-            if(${${sanitizer_variable}})
-                set(SANITIZER_FLAGS "${SANITIZER_FLAGS} -fsanitize-address")
-                message(STATUS "${sanitizer} sanitizer supported using -fsanitize-address")
+if("${ENABLE_SANITIZERS}")
+    #
+    # This appears to indicate that ENABLE_SANITIZERS was set to a
+    # string value that is "one of the true constants", meaning
+    # "1, ON, YES, TRUE, Y, or a non-zero number".
+    #
+    # It does not appear to happen for other settings, including
+    # setting it to a list of one or more sanitizers.
+    #
+    # This setting means "enable all sanitizers that the compiler
+    # supports".
+    #
+    foreach(sanitizer "address" "undefined")
+        # Set -Werror to catch "argument unused during compilation" warnings
+
+        message(STATUS "Checking sanitizer ${sanitizer}")
+        set(sanitizer_variable "sanitize_${sanitizer}")
+        set(CMAKE_REQUIRED_FLAGS "-Werror -fsanitize=${sanitizer}")
+        check_c_compiler_flag("-fsanitize=${sanitizer}" ${sanitizer_variable})
+        if(${${sanitizer_variable}})
+            set(SANITIZER_FLAGS "${SANITIZER_FLAGS} -fsanitize=${sanitizer}")
+            message(STATUS "${sanitizer} sanitizer supported using -fsanitizer=${sanitizer}")
+        else()
+            #
+            # Try the versions supported prior to Clang 3.2.
+            # If the sanitizer is "address", try -fsanitize-address.
+            # If it's "undefined", try -fcatch-undefined-behavior.
+            # Otherwise, give up.
+            #
+            set(sanitizer_variable "OLD_${sanitizer_variable}")
+            if ("${sanitizer}" STREQUAL "address")
+                set(CMAKE_REQUIRED_FLAGS "-Werror -fsanitize-address")
+                check_c_compiler_flag("-fsanitize-address" ${sanitizer_variable})
+                if(${${sanitizer_variable}})
+                    set(SANITIZER_FLAGS "${SANITIZER_FLAGS} -fsanitize-address")
+                    message(STATUS "${sanitizer} sanitizer supported using -fsanitize-address")
+                else()
+                    message(STATUS "${sanitizer} isn't a supported sanitizer")
+                endif()
+            elseif("${sanitizer}" STREQUAL "undefined")
+                set(CMAKE_REQUIRED_FLAGS "-Werror -fcatch-undefined-behavior")
+                check_c_compiler_flag("-fcatch-undefined-behavior" ${sanitizer_variable})
+                if(${${sanitizer_variable}})
+                    set(SANITIZER_FLAGS "${SANITIZER_FLAGS} -fcatch-undefined-behavior")
+                    message(STATUS "${sanitizer} sanitizer supported using catch-undefined-behavior")
+                else()
+                    message(STATUS "${sanitizer} isn't a supported sanitizer")
+                endif()
             else()
-                message(FATAL_ERROR "${sanitizer} isn't a supported sanitizer")
+                message(STATUS "${sanitizer} isn't a supported sanitizer")
             endif()
-        elseif("${sanitizer}" STREQUAL "undefined")
-            set(CMAKE_REQUIRED_FLAGS "-Werror -fcatch-undefined-behavior")
-            check_c_compiler_flag("-fcatch-undefined-behavior" ${sanitizer_variable})
+        endif()
+
+        unset(CMAKE_REQUIRED_FLAGS)
+    endforeach()
+else()
+    #
+    # This appears to indicate that ENABLE_SANITIZERS was either:
+    #
+    #   not set;
+    #   set to a set to a string value that is not "one of the true
+    #   constants", meaning "1, ON, YES, TRUE, Y, or a non-zero number".
+    #
+    # The latter includes setting it to "one of the false constants",
+    # meaning the string "is 0, OFF, NO, FALSE, N, IGNORE, NOTFOUND,
+    # the empty string, or ends in the suffix -NOTFOUND."
+    #
+    # It also includes setting it to a list of one or more sanitizers.
+    #
+    # We want to treat "not set" and "set to one of the false constants"
+    # as meaning "do not enable any sanitizers".
+    #
+    # We want to treat "set to a list of one or more sanitizers" as
+    # meaning "enable all the sanitizers in the list".
+    #
+    # This requires that we distinguish between those two cases.
+    #
+    if(ENABLE_SANITIZERS)
+        #
+        # This appears to indicate that ENABLE_SANITIZERS was set to
+        # a string value that is "not one of the false constants".
+        #
+        # We already know it's "not one of the true constants", so
+        # we treat it as a list of sanitizers.
+        #
+        foreach(sanitizer IN LISTS ENABLE_SANITIZERS)
+            #
+            # Set -Werror to catch "argument unused during compilation"
+            # warnings
+            #
+            message(STATUS "Checking sanitizer ${sanitizer}")
+            set(sanitizer_variable "sanitize_${sanitizer}")
+            set(CMAKE_REQUIRED_FLAGS "-Werror -fsanitize=${sanitizer}")
+            check_c_compiler_flag("-fsanitize=${sanitizer}" ${sanitizer_variable})
             if(${${sanitizer_variable}})
-                set(SANITIZER_FLAGS "${SANITIZER_FLAGS} -fcatch-undefined-behavior")
-                message(STATUS "${sanitizer} sanitizer supported using catch-undefined-behavior")
+                set(SANITIZER_FLAGS "${SANITIZER_FLAGS} -fsanitize=${sanitizer}")
+                message(STATUS "${sanitizer} sanitizer supported using -fsanitizer=${sanitizer}")
             else()
-                message(FATAL_ERROR "${sanitizer} isn't a supported sanitizer")
+                #
+                # Try the versions supported prior to Clang 3.2.
+                # If the sanitizer is "address", try -fsanitize-address.
+                # If it's "undefined", try -fcatch-undefined-behavior.
+                # Otherwise, give up.
+                #
+                set(sanitizer_variable "OLD_${sanitizer_variable}")
+                if ("${sanitizer}" STREQUAL "address")
+                    set(CMAKE_REQUIRED_FLAGS "-Werror -fsanitize-address")
+                    check_c_compiler_flag("-fsanitize-address" ${sanitizer_variable})
+                    if(${${sanitizer_variable}})
+                        set(SANITIZER_FLAGS "${SANITIZER_FLAGS} -fsanitize-address")
+                        message(STATUS "${sanitizer} sanitizer supported using -fsanitize-address")
+                    else()
+                        message(FATAL_ERROR "${sanitizer} isn't a supported sanitizer")
+                    endif()
+                elseif("${sanitizer}" STREQUAL "undefined")
+                    set(CMAKE_REQUIRED_FLAGS "-Werror -fcatch-undefined-behavior")
+                    check_c_compiler_flag("-fcatch-undefined-behavior" ${sanitizer_variable})
+                    if(${${sanitizer_variable}})
+                        set(SANITIZER_FLAGS "${SANITIZER_FLAGS} -fcatch-undefined-behavior")
+                        message(STATUS "${sanitizer} sanitizer supported using catch-undefined-behavior")
+                    else()
+                        message(FATAL_ERROR "${sanitizer} isn't a supported sanitizer")
+                    endif()
+                else()
+                    message(FATAL_ERROR "${sanitizer} isn't a supported sanitizer")
+                endif()
             endif()
-        else()
-            message(FATAL_ERROR "${sanitizer} isn't a supported sanitizer")
-        endif()
-    endif()
 
-    unset(CMAKE_REQUIRED_FLAGS)
-endforeach()
+            unset(CMAKE_REQUIRED_FLAGS)
+        endforeach()
+    else()
+        #
+        # This appears to indicate that ENABLE_SANITIZERS was either:
+        #
+        #   not set;
+        #   set to a value that's "one of the false constants";
+        #
+        # so we don't enable any sanitizers.
+        #
+        message(STATUS "Not enabling sanitizers")
+    endif()
+endif()
 
 if(NOT "${SANITIZER_FLAGS}" STREQUAL "")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -g ${SANITIZER_FLAGS} -fno-omit-frame-pointer -fno-optimize-sibling-calls")