Make RDFs merging a bit more solid and verbose

diff --git a/scripts/get_merged_all.sh b/scripts/get_merged_all.sh
index 548a1c36c1fa84e50d320dbe81002f42d59bc15f..99c9cdda143d4988ee00ae1af33701645721a4b1 100755 (executable)
--- a/scripts/get_merged_all.sh
+++ b/scripts/get_merged_all.sh
@@ -50,6 +50,7 @@ cp "${WDEF_PATH}" "${LAST_ITERATION_PATH}"
 
 for qid in "${WDIDS[@]}"
 do
 
 for qid in "${WDIDS[@]}"
 do

+       echo "Processing element with QID ${qid}…" >&2

        NEW_WDEF_PATH="$(mktemp)"
        "${SCRIPT_DIR}/get_merged_element.sh" "${LAST_ITERATION_PATH}" "${qid}" > "${NEW_WDEF_PATH}"
        
        NEW_WDEF_PATH="$(mktemp)"
        "${SCRIPT_DIR}/get_merged_element.sh" "${LAST_ITERATION_PATH}" "${qid}" > "${NEW_WDEF_PATH}"
        

diff --git a/scripts/rdf.sh b/scripts/rdf.sh
index 76792511a6cd07e8f73a7c7a1d694a19a23aedf1..811a49e6a669be85fa1ee6b4d93136bf1c9b17a4 100644 (file)
--- a/scripts/rdf.sh
+++ b/scripts/rdf.sh
@@ -33,7 +33,10 @@ function cacheRDF() {
        # Create cache dir
        mkdir -p "${RDFS_CACHE_DIR}"
        
        # Create cache dir
        mkdir -p "${RDFS_CACHE_DIR}"
        

-       curl "${rdf_url}" > "${rdf_path}"
+       curl --fail --silent --retry 10 "${rdf_url}" > "${rdf_path}"
+       
+       # Dirty solution to prevent from overflooding Wikidata server
+       sleep 1

        
        echo "${rdf_path}"
 }
        
        echo "${rdf_path}"
 }


@@ -50,7 +53,7 @@ function cacheRDFMaxAge() {
 
        local -r rdf_path="${RDFS_CACHE_DIR}${element_qid}.xml"
        
 
        local -r rdf_path="${RDFS_CACHE_DIR}${element_qid}.xml"
        

-       if [ ! -f "${rdf_path}" ]||[[ $(find "${rdf_path}" -mmin "+${max_age}") ]]
+       if [ ! -s "${rdf_path}" ]||[[ $(find "${rdf_path}" -mmin "+${max_age}") ]]

        then
                cacheRDF "${element_qid}"
        else
        then
                cacheRDF "${element_qid}"
        else