User:M526244/scripts/BASH/procBOM

From OpenStreetMap Wiki
Jump to: navigation, search
#!/bin/bash

 cd ~/GPS/BOM

 #Step 1: download latest BOM stations.zip
 [ -e stations.zip ] ||
  wget ftp://ftp.bom.gov.au/anon2/home/ncc/metadata/sitelists/stations.zip

 #Step 2: unpack it into stations.txt
 [ -e stations.txt ] ||
  unzip stations.zip

 #Step 3: download all OSM monitoring_station nodes
 if ! [ -e monitoring-station.osm ]
 then
  #Step 3a: determine bounding box enclosing all BOM entries
  tail -n +5 stations.txt | head -n -1 >bbox.tmp
  BBlft=$(echo $(cut -c81-89 bbox.tmp | sort -n | head -n 1))
  BBbot=$(echo $(cut -c72-79 bbox.tmp | sort -n | head -n 1))
  BBrgt=$(echo $(cut -c81-89 bbox.tmp | sort -n | tail -n 1))
  BBtop=$(echo $(cut -c72-79 bbox.tmp | sort -n | tail -n 1))
  rm -f bbox.tmp

  wget -O monitoring-station.osm http://xapi.openstreetmap.org/api/0.6/%2A%5Bman_made=monitoring_station%5D%5Bbbox=$BBlft,$BBbot,$BBrgt,$BBtop%5D
 fi

 #Step 4: subdivide out all <node>..</node> XML blocks
 if ! [ -e OSMstations ]
 then
  mkdir OSMstations
  while read
  do
   if echo $REPLY | grep -q "<node "
   then
    OSMid=$(echo $REPLY | sed -e "s/^.* id='\([0-9]*\)'.*$/\1/")
    nodeXML=OSMstations/$OSMid
    echo $REPLY >>$nodeXML
    until echo $REPLY | grep -q "</node>"
    do
     if ! read
     then
      echo "Warning! monitoring-station.osm seems to be truncated." >&2
      exit 1
     else
      echo $REPLY >>$nodeXML
      echo $REPLY | grep -q "<tag k='name'" &&
       echo OSM:monitoring_station:$OSMid:$(echo $REPLY | sed -e "s/^.* v='\(.*\)'.*$/\1/")
     fi
    done
   fi
  done <monitoring-station.osm
 fi

 #Step 5: split out all stations which have a WMO id; any others save along the way.
 maxstn=$(cat stations.txt | wc -l)

 BOMdt=$(sed -e '2!d;s/^.*Produced: \([ ,0-9A-Za-z]*\).*$/\1/' <stations.txt)

 coYEAR=$(date +%Y)

 for ((line=5;line<$maxstn;line++))
 do
  edtcmd=$(echo $line\!d)

  sed -e $edtcmd stations.txt >BOMstn.tmp
  BOMid=$(cut -c2-7 BOMstn.tmp)
  BOMnm=$(cut -c15-54 BOMstn.tmp)
  xmlBOMnm=$(echo $BOMnm | sed -e "s/&/\&amp;/g;s/</\&lt;/g;s/>/\&gt;/g;s/\"/\&quot;/g;s/'/\&apos;/g")
  lat=$(cut -c72-79 BOMstn.tmp)
  lon=$(cut -c81-89 BOMstn.tmp)
  XAPIlon=$(echo "scale=4;if($lon>180) $lon-360 else $lon" | bc)
  endDT=$(cut -c67-70 BOMstn.tmp)
  BARht=$(cut -c121-128 BOMstn.tmp)
  WMOid=$(cut -c130-135 BOMstn.tmp)

  if [ -n "$BOMid" -a -n "$lat" -a -n "$lon" ] && [ "$endDT" = "  .." ] || [ $endDT -ge $coYEAR ]
  then
   if [ "$WMOid" = "    .." ]
   then
    echo "BOM:$BOMid:$BOMnm"
    fragDir=BOMstations/$(echo $BOMid)
    if ! [ -e $fragDir ]
    then
     mkdir -p $fragDir
     cat BOMstn.tmp >>$fragDir/raw
    fi
    if grep -il "<tag k='name' v='$xmlBOMnm'/>" OSMstations/* >capOSMfn
    then
     #Case 6: No WMOid available, but name matches one or more OSM nodes...
     mkdir -p 6.Manual-Check-BOM-OSM-Possible-Name-Match
     mv $(cat capOSMfn) $fragDir
     echo "<tag k='fixme' v='not_reviewed'/>"                                                          >>$fragDir/tags
     echo "<tag k='BOM:man_made' v='monitoring_station'/>"                                             >>$fragDir/tags
     echo "<tag k='BOM:monitoring:weather' v='yes'/>"                                                  >>$fragDir/tags
     [ "$BARht" = "      .." ] ||
      echo "<tag k='BOM:weather:barometer' v='yes'/>"                                                  >>$fragDir/tags
     echo "<tag k='BOM:ID' v='$(echo $BOMid)'/>"                                                       >>$fragDir/tags
     echo "<tag k='attribution' v='With kind permission: Australian Bureau of Meteorology ($BOMdt)'/>" >>$fragDir/tags
     echo "<tag k='attribution:url' v='http://www.bom.gov.au/other/copyright.shtml'/>"          >>$fragDir/tags
     echo "<tag k='BOM:name' v='$xmlBOMnm'/>"                                                          >>$fragDir/tags
     echo "<tag k='BOM:lat' v='$(echo $lat)'/>"                                                        >>$fragDir/tags
     echo "<tag k='BOM:lon' v='$(echo $lon)'/>"                                                        >>$fragDir/tags
     [ "$lon" = "$XAPIlon" ] ||
      echo "<tag k='BOM:XAPIlon' v='$XAPIlon'/>"                                                       >>$fragDir/tags
     echo "<tag k='delete' v='me'/>"                                                                   >>$fragDir/tags
     mv $fragDir 6.Manual-Check-BOM-OSM-Possible-Name-Match
    else
     #Case 5: No WMOid available, name new to OSM...
     mkdir -p 5.Manual-Check-BOM-nonWMO-new-to-OSM
     echo "<node id='-$line' lat='$(echo $lat)' lon='$(echo $XAPIlon)'>"                               >>$fragDir/tags
     echo "<tag k='fixme' v='not_reviewed'/>"                                                          >>$fragDir/tags
     echo "<tag k='name' v='$xmlBOMnm'/>"                                                              >>$fragDir/tags
     echo "<tag k='man_made' v='monitoring_station'/>"                                                 >>$fragDir/tags
     echo "<tag k='monitoring:weather' v='yes'/>"                                                      >>$fragDir/tags
     [ "$BARht" = "      .." ] ||
      echo "<tag k='weather:barometer' v='yes'/>"                                                      >>$fragDir/tags
     echo "<tag k='BOM:ID' v='$(echo $BOMid)'/>"                                                       >>$fragDir/tags
     [ "$lon" = "$XAPIlon" ] ||
      echo "<tag k='BOM:lon' v='$(echo $lon)'/>"                                                       >>$fragDir/tags
     echo "<tag k='attribution' v='With kind permission: Australian Bureau of Meteorology ($BOMdt)'/>" >>$fragDir/tags
     echo "<tag k='attribution:url' v='http://www.bom.gov.au/other/copyright.shtml'/>"          >>$fragDir/tags
     echo "<tag k='delete' v='me'/>"                                                                   >>$fragDir/tags
     echo "</node>"                                                                                    >>$fragDir/tags
     cat $fragDir/tags >>5.automergeBOM-non-WMO-new-to-OSM
     mv $fragDir 5.Manual-Check-BOM-nonWMO-new-to-OSM
    fi
    rm -f capOSMfn
   else
    fragDir=WMOstations/$(echo $WMOid)
    mkdir -p $fragDir
    [ -e $fragDir/raw ] ||
     cat BOMstn.tmp >>$fragDir/raw

    if ! [ -e $fragDir/old-OSM.osm ]
    then
     if grep -l "<tag k='wmo:id' v='$(echo $WMOid)'/>" OSMstations/* >capOSMfn
     then
      mv $(cat capOSMfn) $fragDir/old-OSM.osm
      #rm $(cat capOSMfn)
      #wget -O $fragDir/old-OSM.osm http://xapi.openstreetmap.org/api/0.6/%2A%5Bwmo:id=$(echo $WMOid)%5D
     fi
     rm -f capOSMfn
    fi

    if [ -e $fragDir/old-OSM.osm ]
    then
     OSMid=$(grep "<node" $fragDir/old-OSM.osm | sed -e "s/^.* id='\([0-9]*\)'.*$/\1/") 

     if grep -q "<node .* version='1' changeset='4762601'" $fragDir/old-OSM.osm
     then
      echo "BOM:$BOMid:$BOMnm aka WMO:$(echo $WMOid) aka OSM:$OSMid"

      if ! [ -e $fragDir/new-OSM.osm ]
      then
       oldlat=$(grep "<node" $fragDir/old-OSM.osm | sed -e "s/^.*lat='\([-.0-9]*\)'.*$/\1/")
       oldlon=$(grep "<node" $fragDir/old-OSM.osm | sed -e "s/^.*lon='\([-.0-9]*\)'.*$/\1/")
       grep -q "<tag k='NOAA:lat'" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='NOAA:lat' v='$oldlat'/>"                   >>$fragDir/moddeck
       grep -q "<tag k='NOAA:lon'" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='NOAA:lon' v='$oldlon'/>"                   >>$fragDir/moddeck
       echo "/<node/s/\(^.*lat=\)'\([-.0-9]*\)'\(.*$\)/\1'$(echo $lat)'\3/" >>$fragDir/moddeck
       echo "/<node/s/\(^.*lon=\)'\([-.0-9]*\)'\(.*$\)/\1'$XAPIlon'\3/" >>$fragDir/moddeck
       grep -iq "<tag k='BOM:ID'" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='BOM:ID' v='$(echo $BOMid)'/>"              >>$fragDir/moddeck
       [ "$lon" = "$XAPIlon" ] ||
        grep -iq "<tag k='BOM:lon'" $fragDir/old-OSM.osm ||
         echo "/<\/node>/i<tag k='BOM:lon' v='$(echo $lon)'/>"              >>$fragDir/moddeck
       grep -iq "<tag k='name' v='$xmlBOMnm'/>" $fragDir/old-OSM.osm ||
        grep -iq "<tag k='BOM:name'" $fragDir/old-OSM.osm ||
         echo "/<\/node>/i<tag k='BOM:name' v='$xmlBOMnm'/>"                >>$fragDir/moddeck
       [ "$BARht" = "      .." ] ||
        grep -iq "<tag k='weather:barometer'" $fragDir/old-OSM.osm ||
         echo "/<\/node>/i<tag k='weather:barometer' v='yes'/>"             >>$fragDir/moddeck
       grep -iq "<tag k='attribution'" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='attribution' v='Base data NOAA, updated with kind permission from Australian Bureau of Meteorology ($BOMdt)'/>" >>$fragDir/moddeck
       grep -iq "<tag k='attribution:url'" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='attribution:url' v='http://www.bom.gov.au/other/copyright.shtml'/>" >>$fragDir/moddeck
       grep -q "<tag k='delete' v='me'/>" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='delete' v='me'/>"                          >>$fragDir/moddeck
       sed -f $fragDir/moddeck $fragDir/old-OSM.osm >$fragDir/new-OSM.osm
      fi
      if grep -iq "<tag k='BOM:name'" $fragDir/new-OSM.osm
      then
       #Case 2: WMOid matches OSM node from correct changeset, but name is different...
       cat $fragDir/new-OSM.osm >>2.automergeBOM-OSM-WMO-namecheck
       mkdir -p 2.Auto-Merged-BOM-WMO-OSM-stations-Name-mismatch
       mv $fragDir 2.Auto-Merged-BOM-WMO-OSM-stations-Name-mismatch
      else
       #Case 1: Both WMOid and name match OSM node from correct changeset...
       cat $fragDir/new-OSM.osm >>1.automergeBOM-OSM-WMO
       mkdir -p 1.Auto-Merged-BOM-WMO-OSM-stations
       mv $fragDir 1.Auto-Merged-BOM-WMO-OSM-stations
      fi

     else
      #Case 4: WMOid matches OSM node, but not from correct changeset...
      echo "BOM:$BOMid:$BOMnm aka WMO:$(echo $WMOid) appears in OSM:$OSMid, but not in Changeset 4762601."

      if ! [ -e $fragDir/new-OSM.osm ]
      then
       grep -q "<tag k='BOM:lat'" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='BOM:lat' v='$(echo $lat)'/>"           >>$fragDir/moddeck
       grep -q "<tag k='BOM:lon'" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='BOM:lon' v='$XAPIlon'/>"           >>$fragDir/moddeck
       grep -iq "<tag k='BOM:ID'" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='BOM:ID' v='$(echo $BOMid)'/>"          >>$fragDir/moddeck
       grep -iq "<tag k='name' v='$xmlBOMnm'/>" $fragDir/old-OSM.osm ||
        grep -iq "<tag k='BOM:name'" $fragDir/old-OSM.osm ||
         echo "/<\/node>/i<tag k='BOM:name' v='$xmlBOMnm'/>"            >>$fragDir/moddeck
       [ "$lon" = "$XAPIlon" ] ||
        grep -iq "<tag k='BOM:lon'" $fragDir/old-OSM.osm ||
         echo "/<\/node>/i<tag k='BOM:lon' v='$(echo $lon)'/>"          >>$fragDir/moddeck
       [ "$BARht" = "      .." ] ||
        grep -iq "<tag k='weather:barometer'" $fragDir/old-OSM.osm ||
         echo "/<\/node>/i<tag k='weather:barometer' v='yes'/>"         >>$fragDir/moddeck
       grep -q "<tag k='delete' v='me'/>" $fragDir/old-OSM.osm ||
        echo "/<\/node>/i<tag k='delete' v='me'/>"                      >>$fragDir/moddeck
       sed -f $fragDir/moddeck $fragDir/old-OSM.osm >$fragDir/new-OSM.osm
      fi
      cat $fragDir/new-OSM.osm >>4.automergeBOM-OSMupdated-WMO
      mkdir -p 4.Manual-Check-OSM-Updated
      mv $fragDir 4.Manual-Check-OSM-Updated
     fi
    else
     #Case 3: WMOid available, but no matching OSM node...
     echo "BOM:$BOMid:$BOMnm aka WMO:$(echo $WMOid) does not currently appear in OSM."
     echo "<node id='-$line' lat='$(echo $lat)' lon='$XAPIlon'>"        >>$fragDir/tags
     echo "<tag k='fixme' v='not_reviewed'/>"                           >>$fragDir/tags
     echo "<tag k='name' v='$xmlBOMnm'/>"                               >>$fragDir/tags
     echo "<tag k='man_made' v='monitoring_station'/>"                  >>$fragDir/tags
     echo "<tag k='monitoring:weather' v='yes'/>"                       >>$fragDir/tags
     [ "$BARht" = "      .." ] ||
      echo "<tag k='weather:barometer' v='yes'/>"                       >>$fragDir/tags
     echo "<tag k='BOM:ID' v='$(echo $BOMid)'/>"                        >>$fragDir/tags
     [ "$lon" = "$XAPIlon" ] ||
      echo "<tag k='BOM:lon' v='$(echo $lon)'/>"                        >>$fragDir/tags
     echo "<tag k='wmo:id' v='$(echo $WMOid)'/>"                        >>$fragDir/tags
     echo "<tag k='attribution' v='With kind permission: Australian Bureau of Meteorology ($BOMdt)'/>" >>$fragDir/tags
     echo "<tag k='attribution:url' v='http://www.bom.gov.au/other/copyright.shtml'/>" >>$fragDir/tags
     echo "<tag k='delete' v='me'/>"                                    >>$fragDir/tags
     echo "</node>"                                                     >>$fragDir/tags
     cat $fragDir/tags >>3.automergeBOM-WMO-new-to-OSM
     mkdir -p 3.Autocreated-BOM-WMO-new-to-OSM
     mv $fragDir 3.Autocreated-BOM-WMO-new-to-OSM
    fi
   fi
  fi
  rm -f BOMstn.tmp
 done

 if [ -e 1.automergeBOM-OSM-WMO ]
 then
  cat model/header.osm 1.automergeBOM-OSM-WMO model/footer.osm >1.automergeBOM-OSM-WMO.osm
  rm 1.automergeBOM-OSM-WMO
  echo "Produced 1.automergeBOM-OSM-WMO.osm - please review and submit if acceptable."
 fi

 if [ -e 2.automergeBOM-OSM-WMO-namecheck ]
 then
  cat model/header.osm 2.automergeBOM-OSM-WMO-namecheck model/footer.osm >2.automergeBOM-OSM-WMO-namecheck.osm
  rm 2.automergeBOM-OSM-WMO-namecheck
  echo "Produced 2.automergeBOM-OSM-WMO-namecheck.osm - please review, amend and submit if acceptable."
 fi

 if [ -e 3.automergeBOM-WMO-new-to-OSM ]
 then
  cat model/header.osm 3.automergeBOM-WMO-new-to-OSM model/footer.osm >3.automergeBOM-WMO-new-to-OSM.osm
  rm 3.automergeBOM-WMO-new-to-OSM
  echo "Produced 3.automergeBOM-WMO-new-to-OSM.osm - please review, amend and submit if acceptable."
 fi

 if [ -e 4.automergeBOM-OSMupdated-WMO ]
 then
  cat model/header.osm 4.automergeBOM-OSMupdated-WMO model/footer.osm >4.automergeBOM-OSMupdated-WMO.osm
  rm 4.automergeBOM-OSMupdated-WMO
  echo "Produced 4.automergeBOM-OSMupdated-WMO.osm - please review, amend and submit if acceptable."
 fi

 if [ -e 5.automergeBOM-non-WMO-new-to-OSM ]
 then
  cat model/header.osm 5.automergeBOM-non-WMO-new-to-OSM model/footer.osm >5.automergeBOM-non-WMO-new-to-OSM.osm
  rm 5.automergeBOM-non-WMO-new-to-OSM
  echo "Produced 5.automergeBOM-non-WMO-new-to-OSM.osm - please review, amend and submit if acceptable."
 fi

 rmdir --ignore-fail-on-non-empty ???stations