ExternalData: Add support for multiple hash algorithms

Add support for projects to have `Data.txt.md5` *and* `Data.txt.sha512`
where the content links hold hashes for the same file.  Check all
`ExternalData_URL_TEMPLATES` entries in order for all available hashes.
The data acquisition is considered a failure if none of the available
URL resources has any of the given hashes.  This makes it possible to
have multiple data server resources where all servers do not support all
hashing algorithms.

1 files changed, 91 insertions, 19 deletions

diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake
index b00de144a5..07cd114578 100644
--- a/Modules/ExternalData.cmake
+++ b/Modules/ExternalData.cmake
@@ -201,6 +201,11 @@ For example, the argument ``DATA{img.png}`` may be satisfied by either a
 real ``img.png`` file in the current source directory or a ``img.png.md5``
 file containing its MD5 sum.
 
+Multiple content links of the same name with different hash algorithms
+are supported (e.g. ``img.png.sha256`` and ``img.png.sha1``) so long as
+they all correspond to the same real file.  This allows objects to be
+fetched from sources indexed by different hash algorithms.
+
 Referencing File Series
 """""""""""""""""""""""
 
@@ -429,8 +434,10 @@ function(ExternalData_add_target target)
     string(REPLACE "|" ";" tuple "${entry}")
     list(GET tuple 0 file)
     list(GET tuple 1 name)
-    list(GET tuple 2 ext)
-    set(stamp "${ext}-stamp")
+    list(GET tuple 2 exts)
+    string(REPLACE "+" ";" exts_list "${exts}")
+    list(GET exts_list 0 first_ext)
+    set(stamp "-hash-stamp")
     if(NOT DEFINED "_ExternalData_FILE_${file}")
       set("_ExternalData_FILE_${file}" 1)
       get_property(added DIRECTORY PROPERTY "_ExternalData_FILE_${file}")
@@ -446,12 +453,12 @@ function(ExternalData_add_target target)
           OUTPUT "${file}${stamp}" "${file}"
           # Run the data fetch/update script.
           COMMAND ${CMAKE_COMMAND} -Drelative_top=${CMAKE_BINARY_DIR}
-                                   -Dfile=${file} -Dname=${name} -Dext=${ext}
+                                   -Dfile=${file} -Dname=${name} -Dexts=${exts}
                                    -DExternalData_ACTION=fetch
                                    -DExternalData_CONFIG=${config}
                                    -P ${_ExternalData_SELF}
           # Update whenever the object hash changes.
-          MAIN_DEPENDENCY "${name}${ext}"
+          MAIN_DEPENDENCY "${name}${first_ext}"
           )
       endif()
       list(APPEND files "${file}${stamp}")
@@ -798,6 +805,7 @@ function(_ExternalData_arg_find_files glob pattern regex)
   cmake_policy(SET CMP0009 NEW)
   file(${glob} globbed RELATIVE "${top_src}" "${top_src}/${pattern}*")
   cmake_policy(POP)
+  set(externals_count -1)
   foreach(entry IN LISTS globbed)
     if("x${entry}" MATCHES "^x(.*)(\\.(${_ExternalData_REGEX_EXT}))$")
       set(relname "${CMAKE_MATCH_1}")
@@ -817,7 +825,11 @@ function(_ExternalData_arg_find_files glob pattern regex)
         set(name "${top_src}/${relname}")
         set(file "${top_bin}/${relname}")
         if(alg)
-          list(APPEND external "${file}|${name}|${alg}")
+          if(NOT "${external_${externals_count}_file_name}" STREQUAL "${file}|${name}")
+            math(EXPR externals_count "${externals_count} + 1")
+            set(external_${externals_count}_file_name "${file}|${name}")
+          endif()
+          list(APPEND external_${externals_count}_algs "${alg}")
         elseif(ExternalData_LINK_CONTENT)
           _ExternalData_link_content("${name}" alg)
           list(APPEND external "${file}|${name}|${alg}")
@@ -830,6 +842,14 @@ function(_ExternalData_arg_find_files glob pattern regex)
       endif()
     endif()
   endforeach()
+  if(${externals_count} GREATER -1)
+    foreach(ii RANGE ${externals_count})
+      string(REPLACE ";" "+" algs_delim "${external_${ii}_algs}")
+      list(APPEND external "${external_${ii}_file_name}|${algs_delim}")
+      unset(external_${ii}_algs)
+      unset(external_${ii}_file_name)
+    endforeach()
+  endif()
   set(external "${external}" PARENT_SCOPE)
   set(internal "${internal}" PARENT_SCOPE)
   set(have_original "${have_original}" PARENT_SCOPE)
@@ -947,13 +967,28 @@ function(_ExternalData_custom_fetch key loc file err_var msg_var)
   set("${msg_var}" "${msg}" PARENT_SCOPE)
 endfunction()
 
-function(_ExternalData_download_object name hash algo var_obj)
+function(_ExternalData_get_from_object_store hash algo var_obj var_success)
+  # Search all object stores for an existing object.
+  foreach(dir ${ExternalData_OBJECT_STORES})
+    set(obj "${dir}/${algo}/${hash}")
+    if(EXISTS "${obj}")
+      message(STATUS "Found object: \"${obj}\"")
+      set("${var_obj}" "${obj}" PARENT_SCOPE)
+      set("${var_success}" 1 PARENT_SCOPE)
+      return()
+    endif()
+  endforeach()
+endfunction()
+
+function(_ExternalData_download_object name hash algo var_obj var_success var_errorMsg)
   # Search all object stores for an existing object.
+  set(success 1)
   foreach(dir ${ExternalData_OBJECT_STORES})
     set(obj "${dir}/${algo}/${hash}")
     if(EXISTS "${obj}")
       message(STATUS "Found object: \"${obj}\"")
       set("${var_obj}" "${obj}" PARENT_SCOPE)
+      set("${var_success}" "${success}" PARENT_SCOPE)
       return()
     endif()
   endforeach()
@@ -1008,6 +1043,7 @@ function(_ExternalData_download_object name hash algo var_obj)
   get_filename_component(dir "${name}" PATH)
   set(staged "${dir}/.ExternalData_${algo}_${hash}")
 
+  set(success 1)
   if(found)
     file(RENAME "${tmp}" "${obj}")
     message(STATUS "Downloaded object: \"${obj}\"")
@@ -1018,38 +1054,74 @@ function(_ExternalData_download_object name hash algo var_obj)
     if(NOT tried)
       set(tried "\n  (No ExternalData_URL_TEMPLATES given)")
     endif()
-    message(FATAL_ERROR "Object ${algo}=${hash} not found at:${tried}")
+    set(success 0)
+    set("${var_errorMsg}" "Object ${algo}=${hash} not found at:${tried}" PARENT_SCOPE)
   endif()
 
   set("${var_obj}" "${obj}" PARENT_SCOPE)
+  set("${var_success}" "${success}" PARENT_SCOPE)
 endfunction()
 
 if("${ExternalData_ACTION}" STREQUAL "fetch")
-  foreach(v ExternalData_OBJECT_STORES file name ext)
+  foreach(v ExternalData_OBJECT_STORES file name exts)
     if(NOT DEFINED "${v}")
       message(FATAL_ERROR "No \"-D${v}=\" value provided!")
     endif()
   endforeach()
 
-  file(READ "${name}${ext}" hash)
-  string(STRIP "${hash}" hash)
-
-  if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$")
-    string(TOUPPER "${CMAKE_MATCH_1}" algo)
-  else()
-    message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"")
-  endif()
+  string(REPLACE "+" ";" exts_list "${exts}")
+  set(succeeded 0)
+  set(errorMsg "")
+  set(hash_list )
+  set(algo_list )
+  set(hash )
+  set(algo )
+  foreach(ext ${exts_list})
+    file(READ "${name}${ext}" hash)
+    string(STRIP "${hash}" hash)
+
+    if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$")
+      string(TOUPPER "${CMAKE_MATCH_1}" algo)
+    else()
+      message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"")
+    endif()
 
-  _ExternalData_download_object("${name}" "${hash}" "${algo}" obj)
+    list(APPEND hash_list ${hash})
+    list(APPEND algo_list ${algo})
+  endforeach()
 
+  list(LENGTH exts_list num_extensions)
+  math(EXPR exts_range "${num_extensions} - 1")
+  foreach(ii RANGE 0 ${exts_range})
+    list(GET hash_list ${ii} hash)
+    list(GET algo_list ${ii} algo)
+    _ExternalData_get_from_object_store("${hash}" "${algo}" obj succeeded)
+    if(succeeded)
+      break()
+    endif()
+  endforeach()
+  if(NOT succeeded)
+    foreach(ii RANGE 0 ${exts_range})
+      list(GET hash_list ${ii} hash)
+      list(GET algo_list ${ii} algo)
+      _ExternalData_download_object("${name}" "${hash}" "${algo}"
+        obj succeeded algoErrorMsg)
+      set(errorMsg "${errorMsg}\n${algoErrorMsg}")
+      if(succeeded)
+        break()
+      endif()
+    endforeach()
+  endif()
+  if(NOT succeeded)
+    message(FATAL_ERROR "${errorMsg}")
+  endif()
   # Check if file already corresponds to the object.
-  set(stamp "${ext}-stamp")
+  set(stamp "-hash-stamp")
   set(file_up_to_date 0)
   if(EXISTS "${file}" AND EXISTS "${file}${stamp}")
     file(READ "${file}${stamp}" f_hash)
     string(STRIP "${f_hash}" f_hash)
     if("${f_hash}" STREQUAL "${hash}")
-      #message(STATUS "File already corresponds to object")
       set(file_up_to_date 1)
     endif()
   endif()