diff options
7 files changed, 104 insertions, 19 deletions
diff --git a/Help/release/dev/ExternalData-multiple-hashes.rst b/Help/release/dev/ExternalData-multiple-hashes.rst new file mode 100644 index 0000000000..608a2778b0 --- /dev/null +++ b/Help/release/dev/ExternalData-multiple-hashes.rst @@ -0,0 +1,7 @@ +ExternalData-multiple-hashes +---------------------------- + +* The :module:`ExternalData` module learned to support multiple + content links for one data file using different hashes, e.g. + ``img.png.sha256`` and ``img.png.sha1``. This allows objects + to be fetched from sources indexed by different hash algorithms. diff --git a/Modules/ExternalData.cmake b/Modules/ExternalData.cmake index b00de144a5..07cd114578 100644 --- a/Modules/ExternalData.cmake +++ b/Modules/ExternalData.cmake @@ -201,6 +201,11 @@ For example, the argument ``DATA{img.png}`` may be satisfied by either a real ``img.png`` file in the current source directory or a ``img.png.md5`` file containing its MD5 sum. +Multiple content links of the same name with different hash algorithms +are supported (e.g. ``img.png.sha256`` and ``img.png.sha1``) so long as +they all correspond to the same real file. This allows objects to be +fetched from sources indexed by different hash algorithms. + Referencing File Series """"""""""""""""""""""" @@ -429,8 +434,10 @@ function(ExternalData_add_target target) string(REPLACE "|" ";" tuple "${entry}") list(GET tuple 0 file) list(GET tuple 1 name) - list(GET tuple 2 ext) - set(stamp "${ext}-stamp") + list(GET tuple 2 exts) + string(REPLACE "+" ";" exts_list "${exts}") + list(GET exts_list 0 first_ext) + set(stamp "-hash-stamp") if(NOT DEFINED "_ExternalData_FILE_${file}") set("_ExternalData_FILE_${file}" 1) get_property(added DIRECTORY PROPERTY "_ExternalData_FILE_${file}") @@ -446,12 +453,12 @@ function(ExternalData_add_target target) OUTPUT "${file}${stamp}" "${file}" # Run the data fetch/update script. COMMAND ${CMAKE_COMMAND} -Drelative_top=${CMAKE_BINARY_DIR} - -Dfile=${file} -Dname=${name} -Dext=${ext} + -Dfile=${file} -Dname=${name} -Dexts=${exts} -DExternalData_ACTION=fetch -DExternalData_CONFIG=${config} -P ${_ExternalData_SELF} # Update whenever the object hash changes. - MAIN_DEPENDENCY "${name}${ext}" + MAIN_DEPENDENCY "${name}${first_ext}" ) endif() list(APPEND files "${file}${stamp}") @@ -798,6 +805,7 @@ function(_ExternalData_arg_find_files glob pattern regex) cmake_policy(SET CMP0009 NEW) file(${glob} globbed RELATIVE "${top_src}" "${top_src}/${pattern}*") cmake_policy(POP) + set(externals_count -1) foreach(entry IN LISTS globbed) if("x${entry}" MATCHES "^x(.*)(\\.(${_ExternalData_REGEX_EXT}))$") set(relname "${CMAKE_MATCH_1}") @@ -817,7 +825,11 @@ function(_ExternalData_arg_find_files glob pattern regex) set(name "${top_src}/${relname}") set(file "${top_bin}/${relname}") if(alg) - list(APPEND external "${file}|${name}|${alg}") + if(NOT "${external_${externals_count}_file_name}" STREQUAL "${file}|${name}") + math(EXPR externals_count "${externals_count} + 1") + set(external_${externals_count}_file_name "${file}|${name}") + endif() + list(APPEND external_${externals_count}_algs "${alg}") elseif(ExternalData_LINK_CONTENT) _ExternalData_link_content("${name}" alg) list(APPEND external "${file}|${name}|${alg}") @@ -830,6 +842,14 @@ function(_ExternalData_arg_find_files glob pattern regex) endif() endif() endforeach() + if(${externals_count} GREATER -1) + foreach(ii RANGE ${externals_count}) + string(REPLACE ";" "+" algs_delim "${external_${ii}_algs}") + list(APPEND external "${external_${ii}_file_name}|${algs_delim}") + unset(external_${ii}_algs) + unset(external_${ii}_file_name) + endforeach() + endif() set(external "${external}" PARENT_SCOPE) set(internal "${internal}" PARENT_SCOPE) set(have_original "${have_original}" PARENT_SCOPE) @@ -947,13 +967,28 @@ function(_ExternalData_custom_fetch key loc file err_var msg_var) set("${msg_var}" "${msg}" PARENT_SCOPE) endfunction() -function(_ExternalData_download_object name hash algo var_obj) +function(_ExternalData_get_from_object_store hash algo var_obj var_success) + # Search all object stores for an existing object. + foreach(dir ${ExternalData_OBJECT_STORES}) + set(obj "${dir}/${algo}/${hash}") + if(EXISTS "${obj}") + message(STATUS "Found object: \"${obj}\"") + set("${var_obj}" "${obj}" PARENT_SCOPE) + set("${var_success}" 1 PARENT_SCOPE) + return() + endif() + endforeach() +endfunction() + +function(_ExternalData_download_object name hash algo var_obj var_success var_errorMsg) # Search all object stores for an existing object. + set(success 1) foreach(dir ${ExternalData_OBJECT_STORES}) set(obj "${dir}/${algo}/${hash}") if(EXISTS "${obj}") message(STATUS "Found object: \"${obj}\"") set("${var_obj}" "${obj}" PARENT_SCOPE) + set("${var_success}" "${success}" PARENT_SCOPE) return() endif() endforeach() @@ -1008,6 +1043,7 @@ function(_ExternalData_download_object name hash algo var_obj) get_filename_component(dir "${name}" PATH) set(staged "${dir}/.ExternalData_${algo}_${hash}") + set(success 1) if(found) file(RENAME "${tmp}" "${obj}") message(STATUS "Downloaded object: \"${obj}\"") @@ -1018,38 +1054,74 @@ function(_ExternalData_download_object name hash algo var_obj) if(NOT tried) set(tried "\n (No ExternalData_URL_TEMPLATES given)") endif() - message(FATAL_ERROR "Object ${algo}=${hash} not found at:${tried}") + set(success 0) + set("${var_errorMsg}" "Object ${algo}=${hash} not found at:${tried}" PARENT_SCOPE) endif() set("${var_obj}" "${obj}" PARENT_SCOPE) + set("${var_success}" "${success}" PARENT_SCOPE) endfunction() if("${ExternalData_ACTION}" STREQUAL "fetch") - foreach(v ExternalData_OBJECT_STORES file name ext) + foreach(v ExternalData_OBJECT_STORES file name exts) if(NOT DEFINED "${v}") message(FATAL_ERROR "No \"-D${v}=\" value provided!") endif() endforeach() - file(READ "${name}${ext}" hash) - string(STRIP "${hash}" hash) - - if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$") - string(TOUPPER "${CMAKE_MATCH_1}" algo) - else() - message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"") - endif() + string(REPLACE "+" ";" exts_list "${exts}") + set(succeeded 0) + set(errorMsg "") + set(hash_list ) + set(algo_list ) + set(hash ) + set(algo ) + foreach(ext ${exts_list}) + file(READ "${name}${ext}" hash) + string(STRIP "${hash}" hash) + + if("${ext}" MATCHES "^\\.(${_ExternalData_REGEX_EXT})$") + string(TOUPPER "${CMAKE_MATCH_1}" algo) + else() + message(FATAL_ERROR "Unknown hash algorithm extension \"${ext}\"") + endif() - _ExternalData_download_object("${name}" "${hash}" "${algo}" obj) + list(APPEND hash_list ${hash}) + list(APPEND algo_list ${algo}) + endforeach() + list(LENGTH exts_list num_extensions) + math(EXPR exts_range "${num_extensions} - 1") + foreach(ii RANGE 0 ${exts_range}) + list(GET hash_list ${ii} hash) + list(GET algo_list ${ii} algo) + _ExternalData_get_from_object_store("${hash}" "${algo}" obj succeeded) + if(succeeded) + break() + endif() + endforeach() + if(NOT succeeded) + foreach(ii RANGE 0 ${exts_range}) + list(GET hash_list ${ii} hash) + list(GET algo_list ${ii} algo) + _ExternalData_download_object("${name}" "${hash}" "${algo}" + obj succeeded algoErrorMsg) + set(errorMsg "${errorMsg}\n${algoErrorMsg}") + if(succeeded) + break() + endif() + endforeach() + endif() + if(NOT succeeded) + message(FATAL_ERROR "${errorMsg}") + endif() # Check if file already corresponds to the object. - set(stamp "${ext}-stamp") + set(stamp "-hash-stamp") set(file_up_to_date 0) if(EXISTS "${file}" AND EXISTS "${file}${stamp}") file(READ "${file}${stamp}" f_hash) string(STRIP "${f_hash}" f_hash) if("${f_hash}" STREQUAL "${hash}") - #message(STATUS "File already corresponds to object") set(file_up_to_date 1) endif() endif() diff --git a/Tests/Module/ExternalData/CMakeLists.txt b/Tests/Module/ExternalData/CMakeLists.txt index 1018dd87d3..737e17a68d 100644 --- a/Tests/Module/ExternalData/CMakeLists.txt +++ b/Tests/Module/ExternalData/CMakeLists.txt @@ -45,6 +45,8 @@ ExternalData_Add_Test(Data1 -D Meta=DATA{MetaTop.dat,REGEX:Meta[ABC].dat} -D Directory=DATA{Directory/,A.dat,REGEX:[BC].dat} -D DirRecurse=DATA{DirRecurse/,RECURSE:,A.dat,REGEX:[BC].dat} + -D MultipleAlgorithmNoSHA1=DATA{MultipleAlgorithmNoSHA1.dat} + -D MultipleAlgorithmNoMD5=DATA{MultipleAlgorithmNoMD5.dat} -D "Semicolons=DATA{Data.dat}\\;DATA{Data.dat}" -P ${CMAKE_CURRENT_SOURCE_DIR}/Data1Check.cmake ) diff --git a/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.md5 b/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.md5 new file mode 100644 index 0000000000..a956f36a1f --- /dev/null +++ b/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.md5 @@ -0,0 +1 @@ +29848e54a4d0343f138ab14419b863de diff --git a/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.sha1 b/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.sha1 new file mode 100644 index 0000000000..43a354086a --- /dev/null +++ b/Tests/Module/ExternalData/MultipleAlgorithmNoMD5.dat.sha1 @@ -0,0 +1 @@ +2af59a7022024974f3b8521b7ed8137c996a79f1 diff --git a/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.md5 b/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.md5 new file mode 100644 index 0000000000..1906cbf7ff --- /dev/null +++ b/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.md5 @@ -0,0 +1 @@ +08cfcf221f76ace7b906b312284e73d7 diff --git a/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.sha1 b/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.sha1 new file mode 100644 index 0000000000..65781b2060 --- /dev/null +++ b/Tests/Module/ExternalData/MultipleAlgorithmNoSHA1.dat.sha1 @@ -0,0 +1 @@ +223b134e6e3a9bf34aa7531c009d97cff6b0d8a3 |