ExternalProject: Refactor pre-configure steps to support no-target uses

The mkdir, download, update and patch steps are used by
FetchContent during the configure phase of the main build. Because
these steps need a target, this has so far required a sub-build to be
set up. The changes here factor out the preparation of the scripts
from the creation of the targets, allowing future work to leverage these
steps without a sub-build (see #21703).

As part of the refactoring, some rationalisation of the stamp files,
repository info files and script names was done to make things more
consistent between download methods and step implementations.
Every download method now records its own specific repository info
in a file and that file is a dependency of the download step. The source
directory is also written to that file, so if the SOURCE_DIR changes, the
download will be retriggered (the existing implementation fails in this
scenario). Each download method now also has just one driver script
that implements the whole step (it may pull in other scripts to do its
task though). The patch step gained support for USES_TERMINAL as
a result of generalising the implementation for custom commands.

Fixes: #21748

1 files changed, 187 insertions, 0 deletions

diff --git a/Modules/ExternalProject/download.cmake.in b/Modules/ExternalProject/download.cmake.in
new file mode 100644
index 0000000000..6ef4eb1842
--- /dev/null
+++ b/Modules/ExternalProject/download.cmake.in
@@ -0,0 +1,187 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+cmake_minimum_required(VERSION 3.5)
+
+function(check_file_hash has_hash hash_is_good)
+  if("${has_hash}" STREQUAL "")
+    message(FATAL_ERROR "has_hash Can't be empty")
+  endif()
+
+  if("${hash_is_good}" STREQUAL "")
+    message(FATAL_ERROR "hash_is_good Can't be empty")
+  endif()
+
+  if("@ALGO@" STREQUAL "")
+    # No check
+    set("${has_hash}" FALSE PARENT_SCOPE)
+    set("${hash_is_good}" FALSE PARENT_SCOPE)
+    return()
+  endif()
+
+  set("${has_hash}" TRUE PARENT_SCOPE)
+
+  message(STATUS "verifying file...
+       file='@LOCAL@'")
+
+  file("@ALGO@" "@LOCAL@" actual_value)
+
+  if(NOT "${actual_value}" STREQUAL "@EXPECT_VALUE@")
+    set("${hash_is_good}" FALSE PARENT_SCOPE)
+    message(STATUS "@ALGO@ hash of
+    @LOCAL@
+  does not match expected value
+    expected: '@EXPECT_VALUE@'
+      actual: '${actual_value}'")
+  else()
+    set("${hash_is_good}" TRUE PARENT_SCOPE)
+  endif()
+endfunction()
+
+function(sleep_before_download attempt)
+  if(attempt EQUAL 0)
+    return()
+  endif()
+
+  if(attempt EQUAL 1)
+    message(STATUS "Retrying...")
+    return()
+  endif()
+
+  set(sleep_seconds 0)
+
+  if(attempt EQUAL 2)
+    set(sleep_seconds 5)
+  elseif(attempt EQUAL 3)
+    set(sleep_seconds 5)
+  elseif(attempt EQUAL 4)
+    set(sleep_seconds 15)
+  elseif(attempt EQUAL 5)
+    set(sleep_seconds 60)
+  elseif(attempt EQUAL 6)
+    set(sleep_seconds 90)
+  elseif(attempt EQUAL 7)
+    set(sleep_seconds 300)
+  else()
+    set(sleep_seconds 1200)
+  endif()
+
+  message(STATUS "Retry after ${sleep_seconds} seconds (attempt #${attempt}) ...")
+
+  execute_process(COMMAND "${CMAKE_COMMAND}" -E sleep "${sleep_seconds}")
+endfunction()
+
+if("@LOCAL@" STREQUAL "")
+  message(FATAL_ERROR "LOCAL can't be empty")
+endif()
+
+if("@REMOTE@" STREQUAL "")
+  message(FATAL_ERROR "REMOTE can't be empty")
+endif()
+
+function(download_and_verify)
+  if(EXISTS "@LOCAL@")
+    check_file_hash(has_hash hash_is_good)
+    if(has_hash)
+      if(hash_is_good)
+        message(STATUS
+"File already exists and hash match (skip download):
+  file='@LOCAL@'
+  @ALGO@='@EXPECT_VALUE@'"
+        )
+        return()
+      else()
+        message(STATUS "File already exists but hash mismatch. Removing...")
+        file(REMOVE "@LOCAL@")
+      endif()
+    else()
+      message(STATUS
+"File already exists but no hash specified (use URL_HASH):
+  file='@LOCAL@'
+Old file will be removed and new file downloaded from URL."
+      )
+      file(REMOVE "@LOCAL@")
+    endif()
+  endif()
+
+  set(retry_number 5)
+
+  message(STATUS "Downloading...
+   dst='@LOCAL@'
+   timeout='@TIMEOUT_MSG@'
+   inactivity timeout='@INACTIVITY_TIMEOUT_MSG@'"
+  )
+  set(download_retry_codes 7 6 8 15)
+  set(skip_url_list)
+  set(status_code)
+  foreach(i RANGE ${retry_number})
+    if(status_code IN_LIST download_retry_codes)
+      sleep_before_download(${i})
+    endif()
+    foreach(url @REMOTE@)
+      if(NOT url IN_LIST skip_url_list)
+        message(STATUS "Using src='${url}'")
+
+        @TLS_VERIFY_CODE@
+        @TLS_CAINFO_CODE@
+        @NETRC_CODE@
+        @NETRC_FILE_CODE@
+
+        file(
+          DOWNLOAD
+          "${url}" "@LOCAL@"
+          @SHOW_PROGRESS@
+          @TIMEOUT_ARGS@
+          @INACTIVITY_TIMEOUT_ARGS@
+          STATUS status
+          LOG log
+          @USERPWD_ARGS@
+          @HTTP_HEADERS_ARGS@
+          )
+
+        list(GET status 0 status_code)
+        list(GET status 1 status_string)
+
+        if(status_code EQUAL 0)
+          check_file_hash(has_hash hash_is_good)
+          if(has_hash AND NOT hash_is_good)
+            message(STATUS "Hash mismatch, removing...")
+            file(REMOVE "@LOCAL@")
+          else()
+            message(STATUS "Downloading... done")
+            return()
+          endif()
+        else()
+          string(APPEND logFailedURLs
+"error: downloading '${url}' failed
+        status_code: ${status_code}
+        status_string: ${status_string}
+        log:
+        --- LOG BEGIN ---
+        ${log}
+        --- LOG END ---
+        "
+          )
+        if(NOT status_code IN_LIST download_retry_codes)
+          list(APPEND skip_url_list "${url}")
+          break()
+        endif()
+      endif()
+    endif()
+    endforeach()
+  endforeach()
+
+  message(FATAL_ERROR
+"Each download failed!
+  ${logFailedURLs}
+  "
+  )
+
+endfunction()
+
+download_and_verify()
+
+set(extract_script @extract_script_filename@)
+if(NOT "${extract_script}" STREQUAL "")
+  include(${extract_script})
+endif()