update.sh

   1 #!/bin/bash
   2
   3 # update.sh - Scripts to merge WMO data with Wikidata.
   4 # Copyright (C) 2021  Pierre Choffet
   5 #
   6 # This program is free software: you can redistribute it and/or modify
   7 # it under the terms of version 3 of the GNU General Public License as
   8 # published by the Free Software Foundation.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 set -euo pipefail
  19
  20 # Script cache dir
  21 CACHE_DIR=${CACHE_DIR:-"${HOME}/.cache/wmo_to_wikidata/"}
  22
  23 # Any stations cache older than this (in minutes) will be updated
  24 STATIONS_MAX_AGE=${STATIONS_MAX_AGE:=1440}
  25
  26 # Hardcoded values
  27 OSCAR_STATIONS_URL='https://oscar.wmo.int/surface/rest/api/search/station'
  28 STATIONS_CACHE_PATH="${CACHE_DIR}/stations.xml"
  29 STATIONS_CLEANED_CACHE_PATH="${CACHE_DIR}/stations_cleaned.xml"
  30
  31 # Fail if something is missing
  32 function assertEnvironment() {
  33         for name in curl yq xmlstarlet
  34         do
  35                 if ! type "${name}" > /dev/null 2>&1
  36                 then
  37                         echo "Cannot find ${name}. Exiting"
  38                         exit 1
  39                 fi
  40         done
  41 }
  42
  43 # Update stations cache, if needed
  44 function ensureStationsCache() {
  45         local -r outdated_path=$(find "${STATIONS_CACHE_PATH}" -mmin "+${STATIONS_MAX_AGE}")
  46
  47         if [ ! -f "${STATIONS_CACHE_PATH}" ]||[ "${outdated_path}" != ''  ]
  48         then
  49                 local -r stations_download_path="$(mktemp)"
  50
  51                 mkdir -p "${CACHE_DIR}"
  52                 curl "${OSCAR_STATIONS_URL}" > "${stations_download_path}"
  53                 echo "<?xml version='1.0' encoding='utf-8' ?><stations>$(yq -x --xml-root station .stationSearchResults "${stations_download_path}")</stations>" | xmlstarlet fo -t > "${STATIONS_CACHE_PATH}"
  54                 rm "${stations_download_path}"
  55         fi
  56 }
  57
  58 assertEnvironment
  59 ensureStationsCache
  60
  61 # Clean stations cache for known problems
  62 xmlstarlet tr -s xslts/stations_clean.xslt "${STATIONS_CACHE_PATH}" | xmlstarlet fo -t > "${STATIONS_CLEANED_CACHE_PATH}"
  63
  64 # Validate stations cache
  65 xmlstarlet val -e -q -s schemas/stations.xsd "${STATIONS_CLEANED_CACHE_PATH}"
  66
  67 # Generate, validate, echo and delete WDEF file
  68 WDEF_PATH=$(mktemp)
  69 xmlstarlet tr -s xslts/generate_wdef.xslt "${STATIONS_CLEANED_CACHE_PATH}" > "${WDEF_PATH}"
  70 xmlstarlet val -e -q -s wdef_schemas/wdef.xsd "${WDEF_PATH}"
  71 cat "${WDEF_PATH}"
  72 rm "${WDEF_PATH}"