From 00b136155183ae522ca458e540a8cf29bf525e74 Mon Sep 17 00:00:00 2001 From: typebrook Date: Thu, 27 Feb 2020 16:32:37 +0800 Subject: update --- tools/osm/josm_install.sh | 8 +++++++ tools/osm/osm | 17 +++++++++++++ tools/osm/osm.api.changeset.add | 10 ++++++++ tools/osm/osm.api.changeset.close | 3 +++ tools/osm/osm.api.changeset.create | 21 ++++++++++++++++ tools/osm/osm.api.changeset.update | 7 ++++++ tools/osm/osm.api.changeset.upload | 6 +++++ tools/osm/osm.api.fetch | 6 +++++ tools/osm/osm.api.fetch.full | 5 ++++ tools/osm/osm.api.fetch.history | 6 +++++ tools/osm/osm.api.member.relation | 4 ++++ tools/osm/osm.api.referenced.relation | 6 +++++ tools/osm/osm.api.referenced.way | 5 ++++ tools/osm/osm.api.upload.to | 12 ++++++++++ tools/osm/osm.file.get | 4 ++++ tools/osm/osm.file.get.full | 4 ++++ tools/osm/osm.file.query | 4 ++++ tools/osm/osm.goto | 3 +++ tools/osm/osm.help | 34 ++++++++++++++++++++++++++ tools/osm/osm.list.ids | 3 +++ tools/osm/osm.list.tag | 10 ++++++++ tools/osm/osm.list.tags | 15 ++++++++++++ tools/osm/osm.member.relation | 8 +++++++ tools/osm/osm.osc.by_member | 31 ++++++++++++++++++++++++ tools/osm/osm.osc.by_tag | 45 +++++++++++++++++++++++++++++++++++ tools/osm/osm.osm.remove | 9 +++++++ tools/osm/osm.pbf.update | 39 ++++++++++++++++++++++++++++++ tools/osm/osm.query | 3 +++ tools/osm/sequence_number.sh | 22 +++++++++++++++++ 29 files changed, 350 insertions(+) create mode 100755 tools/osm/josm_install.sh create mode 100644 tools/osm/osm create mode 100755 tools/osm/osm.api.changeset.add create mode 100755 tools/osm/osm.api.changeset.close create mode 100755 tools/osm/osm.api.changeset.create create mode 100755 tools/osm/osm.api.changeset.update create mode 100755 tools/osm/osm.api.changeset.upload create mode 100755 tools/osm/osm.api.fetch create mode 100755 tools/osm/osm.api.fetch.full create mode 100755 tools/osm/osm.api.fetch.history create mode 100755 tools/osm/osm.api.member.relation create mode 100755 tools/osm/osm.api.referenced.relation create mode 100755 tools/osm/osm.api.referenced.way create mode 100755 tools/osm/osm.api.upload.to create mode 100755 tools/osm/osm.file.get create mode 100755 tools/osm/osm.file.get.full create mode 100755 tools/osm/osm.file.query create mode 100755 tools/osm/osm.goto create mode 100755 tools/osm/osm.help create mode 100755 tools/osm/osm.list.ids create mode 100755 tools/osm/osm.list.tag create mode 100755 tools/osm/osm.list.tags create mode 100755 tools/osm/osm.member.relation create mode 100755 tools/osm/osm.osc.by_member create mode 100755 tools/osm/osm.osc.by_tag create mode 100755 tools/osm/osm.osm.remove create mode 100755 tools/osm/osm.pbf.update create mode 100755 tools/osm/osm.query create mode 100755 tools/osm/sequence_number.sh (limited to 'tools/osm') diff --git a/tools/osm/josm_install.sh b/tools/osm/josm_install.sh new file mode 100755 index 0000000..520875a --- /dev/null +++ b/tools/osm/josm_install.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# scripts from https://josm.openstreetmap.de/wiki/Download#Webstart +echo deb https://josm.openstreetmap.de/apt $(lsb_release -sc) universe | sudo tee /etc/apt/sources.list.d/josm.list > /dev/null &&\ +wget -q https://josm.openstreetmap.de/josm-apt.key -O- | sudo apt-key add - &&\ +sudo apt-get update &&\ +sudo apt-get remove josm josm-plugins &&\ +sudo apt-get install josm diff --git a/tools/osm/osm b/tools/osm/osm new file mode 100644 index 0000000..a595fb7 --- /dev/null +++ b/tools/osm/osm @@ -0,0 +1,17 @@ +#! /bin/sh + +#export OSM_SERVER=https://master.apis.dev.openstreetmap.org +export OSM_SERVER=https://api.openstreetmap.org + +export OSM_API=$OSM_SERVER/api/0.6 +export OSM_USER_PASSWD=$(cat $SETTING_DIR/tokens/osm 2>/dev/null) + +FILENAME=$0 + +utils.osm() { + vim $FILENAME && source $FILENAME +} + +osm.api() { + curl -X GET $OSM_API/$1/$2 +} diff --git a/tools/osm/osm.api.changeset.add b/tools/osm/osm.api.changeset.add new file mode 100755 index 0000000..3e4878e --- /dev/null +++ b/tools/osm/osm.api.changeset.add @@ -0,0 +1,10 @@ +#!/bin/sh + +element=$(cat) +header=$(echo $element | grep -E "<(node|way|relation)\s") +ele_type=$(echo $header | sed -r 's/.*<(node|way|relation).*$/\1/') +id=$(echo $header | sed -r 's/.* id=\"([^"]+)\".*$/\1/') + +echo $element | \ +sed -r "s/^( *<(node|way|relation).*version[^ ]+ )(.*)$/\1changeset=\"$1\">/" | \ +curl -X PUT -u $OSM_USER_PASSWD -i -T - $OSM_API/$ele_type/$id diff --git a/tools/osm/osm.api.changeset.close b/tools/osm/osm.api.changeset.close new file mode 100755 index 0000000..3b016c3 --- /dev/null +++ b/tools/osm/osm.api.changeset.close @@ -0,0 +1,3 @@ +#!/bin/sh + +curl -X PUT -u $OSM_USER_PASSWD -i $OSM_API/changeset/$1/close diff --git a/tools/osm/osm.api.changeset.create b/tools/osm/osm.api.changeset.create new file mode 100755 index 0000000..ed2601a --- /dev/null +++ b/tools/osm/osm.api.changeset.create @@ -0,0 +1,21 @@ +#!/bin/sh + +echo -n "type comment: " +read comment + +info=" + + + + + + + " + +changeset_id=$(echo $info |\ + curl -u $OSM_USER_PASSWD --upload-file - $OSM_API/changeset/create |\ + tail -1) + +echo +echo "changeset created, check $OSM_SERVER/changeset/$changeset_id" +echo "$changeset_id is copied into clipboard" diff --git a/tools/osm/osm.api.changeset.update b/tools/osm/osm.api.changeset.update new file mode 100755 index 0000000..7b32741 --- /dev/null +++ b/tools/osm/osm.api.changeset.update @@ -0,0 +1,7 @@ +#!/bin/sh + +echo -n 'type comment: ' +read -r comment + +echo "" | \ +curl -X PUT -u $OSM_USER_PASSWD -i -T - $OSM_API/changeset/$1 diff --git a/tools/osm/osm.api.changeset.upload b/tools/osm/osm.api.changeset.upload new file mode 100755 index 0000000..96bdff4 --- /dev/null +++ b/tools/osm/osm.api.changeset.upload @@ -0,0 +1,6 @@ +#!/bin/sh + +cat $2 |\ +sed -r "/<(node|way|relation)/ s/>/ changeset=\"$1\">/" |\ +tee /dev/tty |\ +curl -X POST -u $OSM_USER_PASSWD -i -T - $OSM_API/changeset/$1/upload diff --git a/tools/osm/osm.api.fetch b/tools/osm/osm.api.fetch new file mode 100755 index 0000000..8460c5f --- /dev/null +++ b/tools/osm/osm.api.fetch @@ -0,0 +1,6 @@ +#! /bin/sh + +# get .osm format data +curl -X GET $OSM_API/$1/$2 |\ +tee /tmp/osm &&\ +echo content of $1 $2 is copied into /tmp/osm > /dev/tty diff --git a/tools/osm/osm.api.fetch.full b/tools/osm/osm.api.fetch.full new file mode 100755 index 0000000..5ee78b8 --- /dev/null +++ b/tools/osm/osm.api.fetch.full @@ -0,0 +1,5 @@ +#! /bin/sh + +curl -X GET $OSM_API/$1/$2/full |\ +tee /tmp/osm &&\ +echo "\n" content of $1 $2 and its members are copied into /tmp/osm > /dev/tty diff --git a/tools/osm/osm.api.fetch.history b/tools/osm/osm.api.fetch.history new file mode 100755 index 0000000..bc837bd --- /dev/null +++ b/tools/osm/osm.api.fetch.history @@ -0,0 +1,6 @@ +#! /bin/sh + +curl -X GET $OSM_API/$1/$2/history |\ +tee /tmp/osm &&\ +echo &&\ +echo "\n" history of $1 $2 are copied into /tmp/osm > /dev/tty diff --git a/tools/osm/osm.api.member.relation b/tools/osm/osm.api.member.relation new file mode 100755 index 0000000..5addef7 --- /dev/null +++ b/tools/osm/osm.api.member.relation @@ -0,0 +1,4 @@ +#! /bin/sh + +osm.api.referenced.relation $1 $2 |\ +osm.member.relation $1 $2 $3 diff --git a/tools/osm/osm.api.referenced.relation b/tools/osm/osm.api.referenced.relation new file mode 100755 index 0000000..2ce0631 --- /dev/null +++ b/tools/osm/osm.api.referenced.relation @@ -0,0 +1,6 @@ +#! /bin/sh + +curl -X GET $OSM_API/$1/$2/relations |\ +tee /tmp/osm &&\ +echo &&\ +echo "\n" relations contain $1 $2 are copied into /tmp/osm > /dev/tty diff --git a/tools/osm/osm.api.referenced.way b/tools/osm/osm.api.referenced.way new file mode 100755 index 0000000..1d84238 --- /dev/null +++ b/tools/osm/osm.api.referenced.way @@ -0,0 +1,5 @@ +#!/bin/bash + +curl -X GET $OSM_API/node/$1/ways |\ +tee /tmp/osm &&\ +echo ways contain node $1 are copied into /tmp/osm > /dev/tty diff --git a/tools/osm/osm.api.upload.to b/tools/osm/osm.api.upload.to new file mode 100755 index 0000000..3979220 --- /dev/null +++ b/tools/osm/osm.api.upload.to @@ -0,0 +1,12 @@ +#! /bin/sh + +# allows multiple elements in osm body +tee /tmp/osm |\ +osm.list.ids |\ +sed 's#.*#osm.extract \0 < /tmp/osm#g' |\ +sed "s/.*/\0 \| osm.api.changeset.add $1/g" |\ +while read -r command +do + echo $command + source<(echo "($command &)") +done diff --git a/tools/osm/osm.file.get b/tools/osm/osm.file.get new file mode 100755 index 0000000..e2b67b4 --- /dev/null +++ b/tools/osm/osm.file.get @@ -0,0 +1,4 @@ +#! /bin/sh + +file=$1; shift +osmium getid $file $@ --output-format=osm diff --git a/tools/osm/osm.file.get.full b/tools/osm/osm.file.get.full new file mode 100755 index 0000000..50ee2f1 --- /dev/null +++ b/tools/osm/osm.file.get.full @@ -0,0 +1,4 @@ +#! /bin/sh + +file=$1; shift +osmium getid $file $@ --output-format=osm --add-referenced diff --git a/tools/osm/osm.file.query b/tools/osm/osm.file.query new file mode 100755 index 0000000..4a01b5b --- /dev/null +++ b/tools/osm/osm.file.query @@ -0,0 +1,4 @@ +#! /bin/sh + +file=$1; shift +osmium tags-filter $file $@ --output-format=osm --omit-referenced diff --git a/tools/osm/osm.goto b/tools/osm/osm.goto new file mode 100755 index 0000000..25bceaa --- /dev/null +++ b/tools/osm/osm.goto @@ -0,0 +1,3 @@ +#!/bin/sh + +xdg-open https://www.openstreetmap.org/$1/$2 diff --git a/tools/osm/osm.help b/tools/osm/osm.help new file mode 100755 index 0000000..aa3b36b --- /dev/null +++ b/tools/osm/osm.help @@ -0,0 +1,34 @@ +#! /bin/sh + + echo ' +COMMANDS: + osm.help + util.osm.edit + + osm.api.changeset.create create a new changeset and return its ID + osm.api.changeset.update [changeset ID] update changeset with new comment + osm.api.changeset.upload [changeset ID] [osc file] upload osc file to the given changeset + osm.api.changeset.add [changeset ID] STDIN=osm element, add it into changeset + osm.api.changeset.close [changeset ID] close the given changeset + osm.api.fetch [element type] [element ID] + osm.api.fetch.full like .fetch, but return with all referenced elements + osm.api.fetch.history [element type] [element ID] + osm.api.member.relation [element type] [element ID] [member type] get first relations reference it as member type + osm.api.referenced.relation [element type] [element ID] get all relations reference this element + osm.api.referenced.way [element type] [element ID] get all ways reference this element + osm.api.upload.to + + osm.file.cat + osm.file.get + osm.file.get.full + osm.file.query + osm.list.ids + osm.list.tag + osm.list.tags + + osm.get.id + osm.query + osm.member.relation + osm.pbf.update + osm.osm.update + ' diff --git a/tools/osm/osm.list.ids b/tools/osm/osm.list.ids new file mode 100755 index 0000000..8e5d870 --- /dev/null +++ b/tools/osm/osm.list.ids @@ -0,0 +1,3 @@ +#! /bin/sh + +sed -nr 's/.*<(node|way|relation) id=\"([^"]+)\".*/\1 \2/p' diff --git a/tools/osm/osm.list.tag b/tools/osm/osm.list.tag new file mode 100755 index 0000000..76649a0 --- /dev/null +++ b/tools/osm/osm.list.tag @@ -0,0 +1,10 @@ +#!/bin/bash + +ele_pattern="(node|way|relation)" +sed -nr "/<$ele_pattern/,/<\/$ele_pattern/ { + / /tmp/osm + +for tag in $@ +do + echo $content |\ + osm.list.tag $tag |\ + paste -d',' /tmp/osm - > /tmp/osm.new &&\ + mv /tmp/osm.new /tmp/osm +done + +cat /tmp/osm +echo "\ntag list is also copied into /tmp/osm" > /dev/tty diff --git a/tools/osm/osm.member.relation b/tools/osm/osm.member.relation new file mode 100755 index 0000000..f8d4598 --- /dev/null +++ b/tools/osm/osm.member.relation @@ -0,0 +1,8 @@ +#! /bin/sh + +sed -nr "//g') + + echo $NEW_MEMBERS + + # print matched element with new tags to .osc file + cat $1 |\ + sed -nr "/<$TYPE id=\"$ID\"/,/<\/$TYPE/ { + /<$TYPE id=\"$ID\"/ { + s/(version=\"[0-9]+\")(.*)/\1>/ + a \ \ \ \ $NEW_MEMBERS + } + p + /<\/$TYPE/ q + }" >> $1.osc +done + +# Add .osc structure for output +sed -ir '1 i + 1 i + $ a + $ a ' $1.osc diff --git a/tools/osm/osm.osc.by_tag b/tools/osm/osm.osc.by_tag new file mode 100755 index 0000000..cc92c1d --- /dev/null +++ b/tools/osm/osm.osc.by_tag @@ -0,0 +1,45 @@ +#!/bin/bash + +# create new tags from input line, for example: +# field1 field2 field3 field4 field5 field6 field7 field8... +# [element type] [element ID] key1_added "value1" key2_added "value2" key3_removed key4_removed... + +# key should not quoted, value must be quoted +# And keys which need to be removed must be placed at the end +while read -r line +do + TYPE=$(echo $line | cut -d ' ' -f1) # field1 is type + ID=$(echo $line | cut -d ' ' -f2) # field2 is ID + + # transform key-value pair into tag format: + # + # keys without values are omitted + NEW_TAGS=$(echo $line |\ + cut -d' ' -f3- |\ + sed -r 's/([^ "]+) (\"[^"]+\")//g; s/>[^"]*$/>/') + + # get regex pattern need to removed from original osm element: + # key1|key2|key3|key4 + TAG_PATTERN=$(echo $line |\ + cut -d' ' -f3- | xargs -n2 echo |\ + cut -d' ' -f1 | paste -s -d'|') + + echo $NEW_TAGS > /dev/tty + + # print matched element with new tags to .osc file + cat $1 |\ + sed -nr "/<$TYPE id=\"$ID\"/,/<\/$TYPE/ { + /<$TYPE id=\"$ID\"/ { + s/(version=\"[0-9]+\")(.*)/\1>/ + a \ \ \ \ $NEW_TAGS + } + /> $1.osc +done + +# Add .osc structure for output +sed -ir '1 i + 1 i + $ a + $ a ' $1.osc diff --git a/tools/osm/osm.osm.remove b/tools/osm/osm.osm.remove new file mode 100755 index 0000000..83d6ea8 --- /dev/null +++ b/tools/osm/osm.osm.remove @@ -0,0 +1,9 @@ +#!/bin/bash + +while read -r line +do + # put element type and element ID into array + array=( $(echo $line) ) + cat $1 |\ + sed -i "/<$array[1] id=\"$array[2]\"/,/<\/$array[1]>/ d" +done diff --git a/tools/osm/osm.pbf.update b/tools/osm/osm.pbf.update new file mode 100755 index 0000000..cd115a7 --- /dev/null +++ b/tools/osm/osm.pbf.update @@ -0,0 +1,39 @@ +#!/bin/bash + +GEOFABRICK_SERVER=http://download.geofabrik.de/asia/taiwan-updates +PBF_FILE=$1 + +# get latest sequence number +echo Fetching the latest sequence number +LATEST_SEQ=$(curl --silent http://download.geofabrik.de/asia/taiwan-updates/state.txt |\ + tail -1 | cut -d'=' -f2) +echo Latest sequence number is $LATEST_SEQ + +# get current sequence number +SEQ=$(osmium fileinfo $PBF_FILE |\ + grep osmosis_replication_sequence_number |\ + cut -d'=' -f2) +echo File sequence number is $SEQ + +# while server has osc file with given sequence number, +# get it and do file update +while + (( SEQ++ )) + [ $SEQ -le $LATEST_SEQ ] +do + mkdir -p changes + SEQ_PATH=$(echo $SEQ | sed -r 's/(.{1})(.{3})/00\1\/\2/') + CHANGE_URL=$GEOFABRICK_SERVER/000/$SEQ_PATH.osc.gz + echo $CHANGE_URL + curl -o changes/$SEQ.osc.gz $CHANGE_URL && \ + osmium apply-changes $PBF_FILE changes/$SEQ.osc.gz \ + --output-header=osmosis_replication_sequence_number=$SEQ \ + --overwrite \ + --output $SEQ.osm.pbf + + mv $PBF_FILE $((SEQ-1)).osm.pbf + mv $SEQ.osm.pbf $PBF_FILE +done + +echo +echo File sequence number is $((SEQ-1)), already the latest one on Geofrbrik diff --git a/tools/osm/osm.query b/tools/osm/osm.query new file mode 100755 index 0000000..8d0b9f7 --- /dev/null +++ b/tools/osm/osm.query @@ -0,0 +1,3 @@ +#!/bin/bash + +osmium tags-filter - $@ --input-format=osm --output-format=osm --omit-referenced diff --git a/tools/osm/sequence_number.sh b/tools/osm/sequence_number.sh new file mode 100755 index 0000000..eb365d0 --- /dev/null +++ b/tools/osm/sequence_number.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# $1 as --hour or --minute, $2 as timestamp +# return the latest sequence number + +case $1 in + # hour difference with Tue Jun 4 03:00:00 UTC 2019 + # sequence number=58940 + --hour) + echo $[($2 - 1559617200)/3600 + 58940] + ;; + + # minute difference with latest planet state file + --minute) + benchmark=benchmark + curl https://planet.openstreetmap.org/replication/minute/state > $benchmark + timeString=$(tail -1 $benchmark | tr -d 'timestamp=\\') + timestamp=$(date -d "$timeString" +%s) + seq=$(sed -n 2p $benchmark | tr -d "sequenceNumber=") + rm $benchmark + echo $[$seq - ($timestamp - $2)/60 - 1 ] +esac -- cgit v1.2.3-70-g09d2