From: Pierre Choffet <peuc@wanadoo.fr>
Date: Thu, 23 Dec 2021 21:18:19 +0000 (-0500)
Subject: Fix Wigos identifier regex and stations cleaner
X-Git-Url: https://git.choffet.net/?a=commitdiff_plain;h=11fe2a0a680c809ec99f34fb5f490c62a787f3bf;p=wmo_to_wikidata.git

Fix Wigos identifier regex and stations cleaner
---

diff --git a/schemas/stations.xsd b/schemas/stations.xsd
index 6a4013b..b66c7c6 100644
--- a/schemas/stations.xsd
+++ b/schemas/stations.xsd
@@ -434,7 +434,7 @@
 	<!-- Types -->
 	<xsd:simpleType name="wigos-id">
 		<xsd:restriction base="xsd:normalizedString">
-			<xsd:pattern value="([0-9]|1[0-4])-(([0-9]|[1-9][0-9]{1,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-4])-){2}[0-9A-Z]{1,16}(-[0-9A-Z]+)?" />
+			<xsd:pattern value="(\d|1[0-4])(-(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-4])){2}-([0-9A-Za-z]{1,16})" />
 		</xsd:restriction>
 	</xsd:simpleType>
 </xsd:schema>
diff --git a/xslts/stations_clean.xslt b/xslts/stations_clean.xslt
index 5971d2e..35e35da 100644
--- a/xslts/stations_clean.xslt
+++ b/xslts/stations_clean.xslt
@@ -19,7 +19,12 @@
 
 	<!-- List known invalid WIGOS IDs in original data -->
 	<xsl:variable name="wigos-ids">
+		<wigos-id invalid-value="41247" />
+		<wigos-id invalid-value="72388" />
 		<wigos-id invalid-value="0-634-0000-0000" />
+		<wigos-id invalid-value="0-858-02560-COL" />
+		<wigos-id invalid-value="1-620-2001-0507" />
+		<wigos-id invalid-value="NaN-NaN-NaN-undefined" />
 	</xsl:variable>
 	
 	<xsl:param name="hardcoded-wigos-id" select="document('')/*/xsl:variable[@name='wigos-ids']/*"/>
@@ -45,14 +50,14 @@
 	
 	<!-- Remove invalid WIGOS identifiers -->
 	<xsl:template match="wigosStationIdentifiers">
-		<xsl:if test="translate(wigosStationIdentifier, '0123456789-', '') = '' and contains(wigosStationIdentifier, '-') and not($hardcoded-wigos-id[@invalid-value = current()/wigosStationIdentifier])">
+		<xsl:if test="translate(wigosStationIdentifier, '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-', '') = '' and contains(wigosStationIdentifier, '-') and not($hardcoded-wigos-id[@invalid-value = current()/wigosStationIdentifier])">
 			<xsl:copy>
 				<xsl:apply-templates select="node()|@*" />
 			</xsl:copy>
 		</xsl:if>
 	</xsl:template>	
 	<xsl:template match="wigosId">
-		<xsl:if test="translate(., '0123456789-', '') = '' and contains(., '-') and not($hardcoded-wigos-id[@invalid-value = current()])">
+		<xsl:if test="translate(., '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-', '') = '' and contains(., '-') and not($hardcoded-wigos-id[@invalid-value = current()])">
 			<xsl:copy>
 				<xsl:apply-templates select="node()|@*" />
 			</xsl:copy>