User:M526244/scripts/BASH/procBOM
Jump to navigation
Jump to search
#!/bin/bash
cd ~/GPS/BOM
#Step 1: download latest BOM stations.zip
[ -e stations.zip ] ||
wget ftp://ftp.bom.gov.au/anon2/home/ncc/metadata/sitelists/stations.zip
#Step 2: unpack it into stations.txt
[ -e stations.txt ] ||
unzip stations.zip
#Step 3: download all OSM monitoring_station nodes
if ! [ -e monitoring-station.osm ]
then
#Step 3a: determine bounding box enclosing all BOM entries
tail -n +5 stations.txt | head -n -1 >bbox.tmp
BBlft=$(echo $(cut -c81-89 bbox.tmp | sort -n | head -n 1))
BBbot=$(echo $(cut -c72-79 bbox.tmp | sort -n | head -n 1))
BBrgt=$(echo $(cut -c81-89 bbox.tmp | sort -n | tail -n 1))
BBtop=$(echo $(cut -c72-79 bbox.tmp | sort -n | tail -n 1))
rm -f bbox.tmp
wget -O monitoring-station.osm http://xapi.openstreetmap.org/api/0.6/%2A%5Bman_made=monitoring_station%5D%5Bbbox=$BBlft,$BBbot,$BBrgt,$BBtop%5D
fi
#Step 4: subdivide out all <node>..</node> XML blocks
if ! [ -e OSMstations ]
then
mkdir OSMstations
while read
do
if echo $REPLY | grep -q "<node "
then
OSMid=$(echo $REPLY | sed -e "s/^.* id='\([0-9]*\)'.*$/\1/")
nodeXML=OSMstations/$OSMid
echo $REPLY >>$nodeXML
until echo $REPLY | grep -q "</node>"
do
if ! read
then
echo "Warning! monitoring-station.osm seems to be truncated." >&2
exit 1
else
echo $REPLY >>$nodeXML
echo $REPLY | grep -q "<tag k='name'" &&
echo OSM:monitoring_station:$OSMid:$(echo $REPLY | sed -e "s/^.* v='\(.*\)'.*$/\1/")
fi
done
fi
done <monitoring-station.osm
fi
#Step 5: split out all stations which have a WMO id; any others save along the way.
maxstn=$(cat stations.txt | wc -l)
BOMdt=$(sed -e '2!d;s/^.*Produced: \([ ,0-9A-Za-z]*\).*$/\1/' <stations.txt)
coYEAR=$(date +%Y)
for ((line=5;line<$maxstn;line++))
do
edtcmd=$(echo $line\!d)
sed -e $edtcmd stations.txt >BOMstn.tmp
BOMid=$(cut -c2-7 BOMstn.tmp)
BOMnm=$(cut -c15-54 BOMstn.tmp)
xmlBOMnm=$(echo $BOMnm | sed -e "s/&/\&/g;s/</\</g;s/>/\>/g;s/\"/\"/g;s/'/\'/g")
lat=$(cut -c72-79 BOMstn.tmp)
lon=$(cut -c81-89 BOMstn.tmp)
XAPIlon=$(echo "scale=4;if($lon>180) $lon-360 else $lon" | bc)
endDT=$(cut -c67-70 BOMstn.tmp)
BARht=$(cut -c121-128 BOMstn.tmp)
WMOid=$(cut -c130-135 BOMstn.tmp)
if [ -n "$BOMid" -a -n "$lat" -a -n "$lon" ] && [ "$endDT" = " .." ] || [ $endDT -ge $coYEAR ]
then
if [ "$WMOid" = " .." ]
then
echo "BOM:$BOMid:$BOMnm"
fragDir=BOMstations/$(echo $BOMid)
if ! [ -e $fragDir ]
then
mkdir -p $fragDir
cat BOMstn.tmp >>$fragDir/raw
fi
if grep -il "<tag k='name' v='$xmlBOMnm'/>" OSMstations/* >capOSMfn
then
#Case 6: No WMOid available, but name matches one or more OSM nodes...
mkdir -p 6.Manual-Check-BOM-OSM-Possible-Name-Match
mv $(cat capOSMfn) $fragDir
echo "<tag k='fixme' v='not_reviewed'/>" >>$fragDir/tags
echo "<tag k='BOM:man_made' v='monitoring_station'/>" >>$fragDir/tags
echo "<tag k='BOM:monitoring:weather' v='yes'/>" >>$fragDir/tags
[ "$BARht" = " .." ] ||
echo "<tag k='BOM:weather:barometer' v='yes'/>" >>$fragDir/tags
echo "<tag k='BOM:ID' v='$(echo $BOMid)'/>" >>$fragDir/tags
echo "<tag k='attribution' v='With kind permission: Australian Bureau of Meteorology ($BOMdt)'/>" >>$fragDir/tags
echo "<tag k='attribution:url' v='http://www.bom.gov.au/other/copyright.shtml'/>" >>$fragDir/tags
echo "<tag k='BOM:name' v='$xmlBOMnm'/>" >>$fragDir/tags
echo "<tag k='BOM:lat' v='$(echo $lat)'/>" >>$fragDir/tags
echo "<tag k='BOM:lon' v='$(echo $lon)'/>" >>$fragDir/tags
[ "$lon" = "$XAPIlon" ] ||
echo "<tag k='BOM:XAPIlon' v='$XAPIlon'/>" >>$fragDir/tags
echo "<tag k='delete' v='me'/>" >>$fragDir/tags
mv $fragDir 6.Manual-Check-BOM-OSM-Possible-Name-Match
else
#Case 5: No WMOid available, name new to OSM...
mkdir -p 5.Manual-Check-BOM-nonWMO-new-to-OSM
echo "<node id='-$line' lat='$(echo $lat)' lon='$(echo $XAPIlon)'>" >>$fragDir/tags
echo "<tag k='fixme' v='not_reviewed'/>" >>$fragDir/tags
echo "<tag k='name' v='$xmlBOMnm'/>" >>$fragDir/tags
echo "<tag k='man_made' v='monitoring_station'/>" >>$fragDir/tags
echo "<tag k='monitoring:weather' v='yes'/>" >>$fragDir/tags
[ "$BARht" = " .." ] ||
echo "<tag k='weather:barometer' v='yes'/>" >>$fragDir/tags
echo "<tag k='BOM:ID' v='$(echo $BOMid)'/>" >>$fragDir/tags
[ "$lon" = "$XAPIlon" ] ||
echo "<tag k='BOM:lon' v='$(echo $lon)'/>" >>$fragDir/tags
echo "<tag k='attribution' v='With kind permission: Australian Bureau of Meteorology ($BOMdt)'/>" >>$fragDir/tags
echo "<tag k='attribution:url' v='http://www.bom.gov.au/other/copyright.shtml'/>" >>$fragDir/tags
echo "<tag k='delete' v='me'/>" >>$fragDir/tags
echo "</node>" >>$fragDir/tags
cat $fragDir/tags >>5.automergeBOM-non-WMO-new-to-OSM
mv $fragDir 5.Manual-Check-BOM-nonWMO-new-to-OSM
fi
rm -f capOSMfn
else
fragDir=WMOstations/$(echo $WMOid)
mkdir -p $fragDir
[ -e $fragDir/raw ] ||
cat BOMstn.tmp >>$fragDir/raw
if ! [ -e $fragDir/old-OSM.osm ]
then
if grep -l "<tag k='wmo:id' v='$(echo $WMOid)'/>" OSMstations/* >capOSMfn
then
mv $(cat capOSMfn) $fragDir/old-OSM.osm
#rm $(cat capOSMfn)
#wget -O $fragDir/old-OSM.osm http://xapi.openstreetmap.org/api/0.6/%2A%5Bwmo:id=$(echo $WMOid)%5D
fi
rm -f capOSMfn
fi
if [ -e $fragDir/old-OSM.osm ]
then
OSMid=$(grep "<node" $fragDir/old-OSM.osm | sed -e "s/^.* id='\([0-9]*\)'.*$/\1/")
if grep -q "<node .* version='1' changeset='4762601'" $fragDir/old-OSM.osm
then
echo "BOM:$BOMid:$BOMnm aka WMO:$(echo $WMOid) aka OSM:$OSMid"
if ! [ -e $fragDir/new-OSM.osm ]
then
oldlat=$(grep "<node" $fragDir/old-OSM.osm | sed -e "s/^.*lat='\([-.0-9]*\)'.*$/\1/")
oldlon=$(grep "<node" $fragDir/old-OSM.osm | sed -e "s/^.*lon='\([-.0-9]*\)'.*$/\1/")
grep -q "<tag k='NOAA:lat'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='NOAA:lat' v='$oldlat'/>" >>$fragDir/moddeck
grep -q "<tag k='NOAA:lon'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='NOAA:lon' v='$oldlon'/>" >>$fragDir/moddeck
echo "/<node/s/\(^.*lat=\)'\([-.0-9]*\)'\(.*$\)/\1'$(echo $lat)'\3/" >>$fragDir/moddeck
echo "/<node/s/\(^.*lon=\)'\([-.0-9]*\)'\(.*$\)/\1'$XAPIlon'\3/" >>$fragDir/moddeck
grep -iq "<tag k='BOM:ID'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='BOM:ID' v='$(echo $BOMid)'/>" >>$fragDir/moddeck
[ "$lon" = "$XAPIlon" ] ||
grep -iq "<tag k='BOM:lon'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='BOM:lon' v='$(echo $lon)'/>" >>$fragDir/moddeck
grep -iq "<tag k='name' v='$xmlBOMnm'/>" $fragDir/old-OSM.osm ||
grep -iq "<tag k='BOM:name'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='BOM:name' v='$xmlBOMnm'/>" >>$fragDir/moddeck
[ "$BARht" = " .." ] ||
grep -iq "<tag k='weather:barometer'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='weather:barometer' v='yes'/>" >>$fragDir/moddeck
grep -iq "<tag k='attribution'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='attribution' v='Base data NOAA, updated with kind permission from Australian Bureau of Meteorology ($BOMdt)'/>" >>$fragDir/moddeck
grep -iq "<tag k='attribution:url'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='attribution:url' v='http://www.bom.gov.au/other/copyright.shtml'/>" >>$fragDir/moddeck
grep -q "<tag k='delete' v='me'/>" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='delete' v='me'/>" >>$fragDir/moddeck
sed -f $fragDir/moddeck $fragDir/old-OSM.osm >$fragDir/new-OSM.osm
fi
if grep -iq "<tag k='BOM:name'" $fragDir/new-OSM.osm
then
#Case 2: WMOid matches OSM node from correct changeset, but name is different...
cat $fragDir/new-OSM.osm >>2.automergeBOM-OSM-WMO-namecheck
mkdir -p 2.Auto-Merged-BOM-WMO-OSM-stations-Name-mismatch
mv $fragDir 2.Auto-Merged-BOM-WMO-OSM-stations-Name-mismatch
else
#Case 1: Both WMOid and name match OSM node from correct changeset...
cat $fragDir/new-OSM.osm >>1.automergeBOM-OSM-WMO
mkdir -p 1.Auto-Merged-BOM-WMO-OSM-stations
mv $fragDir 1.Auto-Merged-BOM-WMO-OSM-stations
fi
else
#Case 4: WMOid matches OSM node, but not from correct changeset...
echo "BOM:$BOMid:$BOMnm aka WMO:$(echo $WMOid) appears in OSM:$OSMid, but not in Changeset 4762601."
if ! [ -e $fragDir/new-OSM.osm ]
then
grep -q "<tag k='BOM:lat'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='BOM:lat' v='$(echo $lat)'/>" >>$fragDir/moddeck
grep -q "<tag k='BOM:lon'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='BOM:lon' v='$XAPIlon'/>" >>$fragDir/moddeck
grep -iq "<tag k='BOM:ID'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='BOM:ID' v='$(echo $BOMid)'/>" >>$fragDir/moddeck
grep -iq "<tag k='name' v='$xmlBOMnm'/>" $fragDir/old-OSM.osm ||
grep -iq "<tag k='BOM:name'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='BOM:name' v='$xmlBOMnm'/>" >>$fragDir/moddeck
[ "$lon" = "$XAPIlon" ] ||
grep -iq "<tag k='BOM:lon'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='BOM:lon' v='$(echo $lon)'/>" >>$fragDir/moddeck
[ "$BARht" = " .." ] ||
grep -iq "<tag k='weather:barometer'" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='weather:barometer' v='yes'/>" >>$fragDir/moddeck
grep -q "<tag k='delete' v='me'/>" $fragDir/old-OSM.osm ||
echo "/<\/node>/i<tag k='delete' v='me'/>" >>$fragDir/moddeck
sed -f $fragDir/moddeck $fragDir/old-OSM.osm >$fragDir/new-OSM.osm
fi
cat $fragDir/new-OSM.osm >>4.automergeBOM-OSMupdated-WMO
mkdir -p 4.Manual-Check-OSM-Updated
mv $fragDir 4.Manual-Check-OSM-Updated
fi
else
#Case 3: WMOid available, but no matching OSM node...
echo "BOM:$BOMid:$BOMnm aka WMO:$(echo $WMOid) does not currently appear in OSM."
echo "<node id='-$line' lat='$(echo $lat)' lon='$XAPIlon'>" >>$fragDir/tags
echo "<tag k='fixme' v='not_reviewed'/>" >>$fragDir/tags
echo "<tag k='name' v='$xmlBOMnm'/>" >>$fragDir/tags
echo "<tag k='man_made' v='monitoring_station'/>" >>$fragDir/tags
echo "<tag k='monitoring:weather' v='yes'/>" >>$fragDir/tags
[ "$BARht" = " .." ] ||
echo "<tag k='weather:barometer' v='yes'/>" >>$fragDir/tags
echo "<tag k='BOM:ID' v='$(echo $BOMid)'/>" >>$fragDir/tags
[ "$lon" = "$XAPIlon" ] ||
echo "<tag k='BOM:lon' v='$(echo $lon)'/>" >>$fragDir/tags
echo "<tag k='wmo:id' v='$(echo $WMOid)'/>" >>$fragDir/tags
echo "<tag k='attribution' v='With kind permission: Australian Bureau of Meteorology ($BOMdt)'/>" >>$fragDir/tags
echo "<tag k='attribution:url' v='http://www.bom.gov.au/other/copyright.shtml'/>" >>$fragDir/tags
echo "<tag k='delete' v='me'/>" >>$fragDir/tags
echo "</node>" >>$fragDir/tags
cat $fragDir/tags >>3.automergeBOM-WMO-new-to-OSM
mkdir -p 3.Autocreated-BOM-WMO-new-to-OSM
mv $fragDir 3.Autocreated-BOM-WMO-new-to-OSM
fi
fi
fi
rm -f BOMstn.tmp
done
if [ -e 1.automergeBOM-OSM-WMO ]
then
cat model/header.osm 1.automergeBOM-OSM-WMO model/footer.osm >1.automergeBOM-OSM-WMO.osm
rm 1.automergeBOM-OSM-WMO
echo "Produced 1.automergeBOM-OSM-WMO.osm - please review and submit if acceptable."
fi
if [ -e 2.automergeBOM-OSM-WMO-namecheck ]
then
cat model/header.osm 2.automergeBOM-OSM-WMO-namecheck model/footer.osm >2.automergeBOM-OSM-WMO-namecheck.osm
rm 2.automergeBOM-OSM-WMO-namecheck
echo "Produced 2.automergeBOM-OSM-WMO-namecheck.osm - please review, amend and submit if acceptable."
fi
if [ -e 3.automergeBOM-WMO-new-to-OSM ]
then
cat model/header.osm 3.automergeBOM-WMO-new-to-OSM model/footer.osm >3.automergeBOM-WMO-new-to-OSM.osm
rm 3.automergeBOM-WMO-new-to-OSM
echo "Produced 3.automergeBOM-WMO-new-to-OSM.osm - please review, amend and submit if acceptable."
fi
if [ -e 4.automergeBOM-OSMupdated-WMO ]
then
cat model/header.osm 4.automergeBOM-OSMupdated-WMO model/footer.osm >4.automergeBOM-OSMupdated-WMO.osm
rm 4.automergeBOM-OSMupdated-WMO
echo "Produced 4.automergeBOM-OSMupdated-WMO.osm - please review, amend and submit if acceptable."
fi
if [ -e 5.automergeBOM-non-WMO-new-to-OSM ]
then
cat model/header.osm 5.automergeBOM-non-WMO-new-to-OSM model/footer.osm >5.automergeBOM-non-WMO-new-to-OSM.osm
rm 5.automergeBOM-non-WMO-new-to-OSM
echo "Produced 5.automergeBOM-non-WMO-new-to-OSM.osm - please review, amend and submit if acceptable."
fi
rmdir --ignore-fail-on-non-empty ???stations