Rename tools into bin

31 files changed, 439 insertions, 0 deletions

diff --git a/bin/osm/josm_install.sh b/bin/osm/josm_install.sh
new file mode 100755
index 0000000..520875a
--- /dev/null
+++ b/bin/osm/josm_install.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# scripts from https://josm.openstreetmap.de/wiki/Download#Webstart
+echo deb https://josm.openstreetmap.de/apt $(lsb_release -sc) universe | sudo tee /etc/apt/sources.list.d/josm.list > /dev/null &&\
+wget -q https://josm.openstreetmap.de/josm-apt.key -O- | sudo apt-key add - &&\
+sudo apt-get update &&\
+sudo apt-get remove josm josm-plugins &&\
+sudo apt-get install josm
diff --git a/bin/osm/note b/bin/osm/note
new file mode 100644
index 0000000..aa51031
--- /dev/null
+++ b/bin/osm/note
@@ -0,0 +1 @@
+diff -y <(ls|sort) <(osm.help | cut -d' ' -f3 | sort | uniq)
diff --git a/bin/osm/osm b/bin/osm/osm
new file mode 100644
index 0000000..344f906
--- /dev/null
+++ b/bin/osm/osm
@@ -0,0 +1,18 @@
+#! /bin/sh
+
+export OSM_TEST_SERVER=https://master.apis.dev.openstreetmap.org
+export OSM_SERVER=https://api.openstreetmap.org
+export OSM_TES_SERVER=https://api.openstreetmap.org
+
+export OSM_API=$OSM_SERVER/api/0.6
+export OSM_USER_PASSWD=$(cat $SETTING_DIR/tokens/osm 2>/dev/null)
+
+FILENAME=$0
+
+utils.osm() {
+    vim $FILENAME && source $FILENAME
+}
+
+osm.api() {
+    curl -X GET $OSM_API/$1/$2
+}
diff --git a/bin/osm/osm.api.changeset.add b/bin/osm/osm.api.changeset.add
new file mode 100755
index 0000000..3e4878e
--- /dev/null
+++ b/bin/osm/osm.api.changeset.add
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+element=$(cat)
+header=$(echo $element | grep -E "<(node|way|relation)\s")
+ele_type=$(echo $header | sed -r 's/.*<(node|way|relation).*$/\1/')
+id=$(echo $header | sed -r 's/.* id=\"([^"]+)\".*$/\1/')
+
+echo $element | \
+sed -r "s/^( *<(node|way|relation).*version[^ ]+ )(.*)$/\1changeset=\"$1\">/" | \
+curl -X PUT -u $OSM_USER_PASSWD -i -T - $OSM_API/$ele_type/$id
diff --git a/bin/osm/osm.api.changeset.close b/bin/osm/osm.api.changeset.close
new file mode 100755
index 0000000..3b016c3
--- /dev/null
+++ b/bin/osm/osm.api.changeset.close
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+curl -X PUT -u $OSM_USER_PASSWD -i $OSM_API/changeset/$1/close
diff --git a/bin/osm/osm.api.changeset.commit b/bin/osm/osm.api.changeset.commit
new file mode 100755
index 0000000..6b67833
--- /dev/null
+++ b/bin/osm/osm.api.changeset.commit
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -e
+shopt -s lastpipe
+
+OSM_SERVER=https://api.openstreetmap.org
+OSM_TEST_SERVER=https://master.apis.dev.openstreetmap.org
+if [[ $@ =~ '--serious' ]]; then
+  SERVER=$OSM_SERVER
+else
+  SERVER=$OSM_TEST_SERVER
+fi
+
+OSM_API=${SERVER}/api/0.6
+FILE=${@//--serious/}
+
+# Prompt for comment and User:Password
+if [[ ! -t 0 ]]; then
+  comment=$(cat)
+else
+  read -e -p 'Type comment: ' -r comment </dev/tty
+fi
+if [ -z ${OSM_USER_PASSWD} ]; then
+  read -e -p 'Type USER:PASSWD: ' -r OSM_USER_PASSWD </dev/tty
+fi
+
+create_changeset() {
+  SOURCE_TAG="${SOURCE:+$(printf "<tag k='source' v='%s'/>" $SOURCE)}"
+
+  curl ${OSM_API}/changeset/create \
+    --user ${OSM_USER_PASSWD} \
+    --upload-file - \
+    --silent \
+    <<EOF | tail -1
+<osm>
+  <changeset>
+    ${SOURCE_TAG}
+    <tag k='comment' v='${comment}'/>
+    <tag k='created_by' v='bash script'/>
+    <tag k='bot' v='yes'/>
+  </changeset>
+</osm>
+EOF
+}
+
+# Return http code after uploading a file
+uploade_file_to_changeset() {
+  curl -X POST $OSM_API/changeset/${changeset_id}/upload \
+    --user ${OSM_USER_PASSWD} -i \
+    --upload-file - \
+    --silent -o /dev/null -w "%{http_code}" 
+}
+
+close_changeset() {
+  curl -X PUT ${OSM_API}/changeset/${changeset_id}/close \
+    --user ${OSM_USER_PASSWD} -i \
+    --silent -o /dev/null -w "%{http_code}" 
+}
+
+# Create changeset with given information
+changeset_id=$(create_changeset)
+
+# Print created changeset id
+echo "Changeset created, check ${SERVER}/changeset/${changeset_id}" >/dev/tty
+echo ${changeset_id}
+
+# Upload OSC file to Changeset
+sed -Ee "/<(node|way|relation)/ s/>/ changeset=\"${changeset_id}\">/" $FILE |\
+uploade_file_to_changeset | if [[ $(cat) == '200' ]]; then 
+  echo Upload file $FILE to changeset ${changeset_id} >/dev/tty
+else
+  exit 1
+fi
+
+# Close Changeset
+close_changeset | if [[ $(cat) == '200' ]]; then 
+  echo Changeset ${changeset_id} closed >/dev/tty
+else
+  exit 1
+fi 
diff --git a/bin/osm/osm.api.changeset.create b/bin/osm/osm.api.changeset.create
new file mode 100755
index 0000000..ab9fe85
--- /dev/null
+++ b/bin/osm/osm.api.changeset.create
@@ -0,0 +1,25 @@
+#!/bin/sh
+
+set -e
+
+read -e -p 'Type comment: ' -r comment </dev/tty
+if [ -z $OSM_USER_PASSWD ]; then
+  read -e -p 'Type USER:PASSWD: ' -r OSM_USER_PASSWD </dev/tty
+fi
+
+info="<osm>
+        <changeset>
+          <tag k='comment' v='$comment'/>
+          <tag k='created_by' v='bash script'/>
+          <tag k='bot' v='yes'/>
+        </changeset>
+      </osm>
+     "
+
+changeset_id=$(echo $info |\
+               curl -u $OSM_USER_PASSWD --upload-file - $OSM_API/changeset/create |\
+               tail -1)
+
+echo >/dev/tty
+echo "changeset created, check $OSM_SERVER/changeset/$changeset_id" >/dev/tty
+echo $changeset_id | tee /tmp/changeset && echo copied to /tmp/changeset
diff --git a/bin/osm/osm.api.changeset.update b/bin/osm/osm.api.changeset.update
new file mode 100755
index 0000000..7267449
--- /dev/null
+++ b/bin/osm/osm.api.changeset.update
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+ID=${1:-$(cat /tmp/changeset 2>/dev/null)}
+[ -z $ID ] && echo Please specify Changeset ID && exit 1
+
+read -e -p 'Type comment: ' -r comment </dev/tty
+
+echo "<osm><changeset><tag k=\"comment\" v=\"$comment\"/></changeset></osm>" | \
+curl -X PUT -u $OSM_USER_PASSWD -i -T - $OSM_API/changeset/$1
diff --git a/bin/osm/osm.api.changeset.upload b/bin/osm/osm.api.changeset.upload
new file mode 100755
index 0000000..96bdff4
--- /dev/null
+++ b/bin/osm/osm.api.changeset.upload
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+cat $2 |\
+sed -r "/<(node|way|relation)/ s/>/ changeset=\"$1\">/" |\
+tee /dev/tty |\
+curl -X POST -u $OSM_USER_PASSWD -i -T - $OSM_API/changeset/$1/upload
diff --git a/bin/osm/osm.api.fetch b/bin/osm/osm.api.fetch
new file mode 100755
index 0000000..8460c5f
--- /dev/null
+++ b/bin/osm/osm.api.fetch
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+# get .osm format data
+curl -X GET $OSM_API/$1/$2 |\
+tee /tmp/osm &&\
+echo content of $1 $2 is copied into /tmp/osm > /dev/tty
diff --git a/bin/osm/osm.api.fetch.full b/bin/osm/osm.api.fetch.full
new file mode 100755
index 0000000..5ee78b8
--- /dev/null
+++ b/bin/osm/osm.api.fetch.full
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+curl -X GET $OSM_API/$1/$2/full |\
+tee /tmp/osm &&\
+echo "\n" content of $1 $2 and its members are copied into /tmp/osm > /dev/tty
diff --git a/bin/osm/osm.api.fetch.history b/bin/osm/osm.api.fetch.history
new file mode 100755
index 0000000..bc837bd
--- /dev/null
+++ b/bin/osm/osm.api.fetch.history
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+curl -X GET $OSM_API/$1/$2/history |\
+tee /tmp/osm &&\
+echo &&\
+echo "\n" history of $1 $2 are copied into /tmp/osm > /dev/tty
diff --git a/bin/osm/osm.api.member.relation b/bin/osm/osm.api.member.relation
new file mode 100755
index 0000000..5addef7
--- /dev/null
+++ b/bin/osm/osm.api.member.relation
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+osm.api.referenced.relation $1 $2 |\
+osm.member.relation $1 $2 $3
diff --git a/bin/osm/osm.api.referenced.relation b/bin/osm/osm.api.referenced.relation
new file mode 100755
index 0000000..2ce0631
--- /dev/null
+++ b/bin/osm/osm.api.referenced.relation
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+curl -X GET $OSM_API/$1/$2/relations |\
+tee /tmp/osm &&\
+echo &&\
+echo "\n" relations contain $1 $2 are copied into /tmp/osm > /dev/tty
diff --git a/bin/osm/osm.api.referenced.way b/bin/osm/osm.api.referenced.way
new file mode 100755
index 0000000..1d84238
--- /dev/null
+++ b/bin/osm/osm.api.referenced.way
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+curl -X GET $OSM_API/node/$1/ways |\
+tee /tmp/osm &&\
+echo ways contain node $1 are copied into /tmp/osm > /dev/tty
diff --git a/bin/osm/osm.api.upload.to b/bin/osm/osm.api.upload.to
new file mode 100755
index 0000000..3979220
--- /dev/null
+++ b/bin/osm/osm.api.upload.to
@@ -0,0 +1,12 @@
+#! /bin/sh
+
+# allows multiple elements in osm body
+tee /tmp/osm |\
+osm.list.ids |\
+sed 's#.*#osm.extract \0 < /tmp/osm#g' |\
+sed "s/.*/\0 \| osm.api.changeset.add $1/g" |\
+while read -r command
+do
+    echo $command
+    source<(echo "($command &)")
+done
diff --git a/bin/osm/osm.file.get b/bin/osm/osm.file.get
new file mode 100755
index 0000000..e2b67b4
--- /dev/null
+++ b/bin/osm/osm.file.get
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+file=$1; shift
+osmium getid $file $@ --output-format=osm
diff --git a/bin/osm/osm.file.get.full b/bin/osm/osm.file.get.full
new file mode 100755
index 0000000..50ee2f1
--- /dev/null
+++ b/bin/osm/osm.file.get.full
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+file=$1; shift
+osmium getid $file $@ --output-format=osm --add-referenced
diff --git a/bin/osm/osm.file.query b/bin/osm/osm.file.query
new file mode 100755
index 0000000..4a01b5b
--- /dev/null
+++ b/bin/osm/osm.file.query
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+file=$1; shift
+osmium tags-filter $file $@ --output-format=osm --omit-referenced
diff --git a/bin/osm/osm.goto b/bin/osm/osm.goto
new file mode 100755
index 0000000..25bceaa
--- /dev/null
+++ b/bin/osm/osm.goto
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+xdg-open https://www.openstreetmap.org/$1/$2
diff --git a/bin/osm/osm.help b/bin/osm/osm.help
new file mode 100755
index 0000000..f4c20d2
--- /dev/null
+++ b/bin/osm/osm.help
@@ -0,0 +1,33 @@
+#! /bin/sh
+
+cat <<EOF
+COMMANDS:
+  osm.help
+  util.osm.edit
+  osm.api.changeset.create     create a new changeset and return its ID
+  osm.api.changeset.update     <changeset ID> update changeset with new comment
+  osm.api.changeset.upload     <changeset ID> <osc file> upload osc file to the given changeset
+  osm.api.changeset.add        <changeset ID> STDIN=osm element, add it into changeset
+  osm.api.changeset.close      <changeset ID> close the given changeset
+  osm.api.fetch                <element type> <element ID>
+  osm.api.fetch.full           like .fetch, but return with all referenced elements
+  osm.api.fetch.history        <element type> <element ID>
+  osm.api.member.relation      <element type> <element ID> <member type> get first relations reference it as member type
+  osm.api.referenced.relation  <element type> <element ID> get all relations reference this element
+  osm.api.referenced.way       <element type> <element ID> get all ways reference this element
+  osm.api.upload.to
+
+  osm.file.cat
+  osm.file.get
+  osm.file.get.full
+  osm.file.query
+  osm.list.ids
+  osm.list.tag
+  osm.list.tags
+
+  osm.get.id
+  osm.query
+  osm.member.relation
+  osm.pbf.update
+  osm.osm.update
+EOF
diff --git a/bin/osm/osm.list.ids b/bin/osm/osm.list.ids
new file mode 100755
index 0000000..8e5d870
--- /dev/null
+++ b/bin/osm/osm.list.ids
@@ -0,0 +1,3 @@
+#! /bin/sh
+
+sed -nr 's/.*<(node|way|relation) id=\"([^"]+)\".*/\1 \2/p'
diff --git a/bin/osm/osm.list.tag b/bin/osm/osm.list.tag
new file mode 100755
index 0000000..76649a0
--- /dev/null
+++ b/bin/osm/osm.list.tag
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+ele_pattern="(node|way|relation)"
+sed -nr "/<$ele_pattern/,/<\/$ele_pattern/ {
+    /<tag k=\"$1\"/ {
+        s/.*v=\"([^\"]+)\".*/\1/
+        h
+    }
+    /<\/$ele_pattern/ {x;p;s/.*//;x}
+}"
diff --git a/bin/osm/osm.list.tags b/bin/osm/osm.list.tags
new file mode 100755
index 0000000..767d32a
--- /dev/null
+++ b/bin/osm/osm.list.tags
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+content=$(cat)
+echo $content | osm.list.ids | tr ' ' ',' > /tmp/osm
+
+for tag in $@
+do
+    echo $content |\
+    osm.list.tag $tag |\
+    paste -d',' /tmp/osm - > /tmp/osm.new &&\
+    mv /tmp/osm.new /tmp/osm
+done
+
+cat /tmp/osm
+echo "\ntag list is also copied into /tmp/osm" > /dev/tty
diff --git a/bin/osm/osm.member.relation b/bin/osm/osm.member.relation
new file mode 100755
index 0000000..f8d4598
--- /dev/null
+++ b/bin/osm/osm.member.relation
@@ -0,0 +1,8 @@
+#! /bin/sh
+
+sed -nr "/<relation/,/<\/relation/ {
+    /<relation/ {s/.* id=\"([^\"]+)\".*/\1/;h}
+    /<member type=\"$1\" ref=\"$2\" role=\"$3\"/ {g;p;q}
+}
+$ {s/.*//;P}
+" | head -1
diff --git a/bin/osm/osm.osc.by_member b/bin/osm/osm.osc.by_member
new file mode 100755
index 0000000..5f9d9be
--- /dev/null
+++ b/bin/osm/osm.osc.by_member
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+while read -r line
+do
+
+    TYPE=$(echo $line | cut -d ' ' -f1) # field1 is type
+    ID=$(echo $line | cut -d ' ' -f2)   # field2 is ID
+
+    NEW_MEMBERS=$(echo $line |\
+               cut -d' ' -f3- |\
+               sed -r 's/([0-9]+)/<member type=\"relation\" ref=\"\1\" role=\"subarea\"\/>/g')
+
+    echo $NEW_MEMBERS
+
+    # print matched element with new tags to .osc file
+    cat $1 |\
+    sed -nr "/<$TYPE id=\"$ID\"/,/<\/$TYPE/ {
+                 /<$TYPE id=\"$ID\"/ {
+                     s/(version=\"[0-9]+\")(.*)/\1>/
+                     a \ \ \ \ $NEW_MEMBERS
+                 }
+                 p
+                 /<\/$TYPE/ q
+            }" >> $1.osc
+done
+
+# Add .osc structure for output
+sed -ir '1 i <osmChange version="0.6" generator="bash script">
+         1 i <modify>
+         $ a </modify>
+         $ a </osmChange>' $1.osc
diff --git a/bin/osm/osm.osc.by_tag b/bin/osm/osm.osc.by_tag
new file mode 100755
index 0000000..cc92c1d
--- /dev/null
+++ b/bin/osm/osm.osc.by_tag
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# create new tags from input line, for example: 
+# field1         field2       field3     field4   field5     field6   field7       field8...
+# [element type] [element ID] key1_added "value1" key2_added "value2" key3_removed key4_removed...
+
+# key should not quoted, value must be quoted
+# And keys which need to be removed must be placed at the end
+while read -r line
+do
+    TYPE=$(echo $line | cut -d ' ' -f1) # field1 is type
+    ID=$(echo $line | cut -d ' ' -f2)   # field2 is ID
+
+    # transform key-value pair into tag format:
+    # <tag k="[key]" v="[value]"/>
+    # keys without values are omitted
+    NEW_TAGS=$(echo $line |\
+               cut -d' ' -f3- |\
+               sed -r 's/([^ "]+) (\"[^"]+\")/<tag k=\"\1\" v=\2\/>/g; s/>[^"]*$/>/')
+
+    # get regex pattern need to removed from original osm element:
+    # key1|key2|key3|key4
+    TAG_PATTERN=$(echo $line |\
+                  cut -d' ' -f3- | xargs -n2 echo |\
+                  cut -d' ' -f1 | paste -s -d'|')
+
+    echo $NEW_TAGS > /dev/tty
+
+    # print matched element with new tags to .osc file
+    cat $1 |\
+    sed -nr "/<$TYPE id=\"$ID\"/,/<\/$TYPE/ {
+                 /<$TYPE id=\"$ID\"/ {
+                     s/(version=\"[0-9]+\")(.*)/\1>/
+                     a \ \ \ \ $NEW_TAGS
+                 }
+                 /<tag k=\"($TAG_PATTERN)\"/ !p
+                 /<\/$TYPE/ q
+            }" >> $1.osc
+done
+
+# Add .osc structure for output
+sed -ir '1 i <osmChange version="0.6" generator="bash script">
+         1 i <modify>
+         $ a </modify>
+         $ a </osmChange>' $1.osc
diff --git a/bin/osm/osm.osm.remove b/bin/osm/osm.osm.remove
new file mode 100755
index 0000000..83d6ea8
--- /dev/null
+++ b/bin/osm/osm.osm.remove
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+while read -r line
+do
+    # put element type and element ID into array
+    array=( $(echo $line) )
+    cat $1 |\
+    sed -i "/<$array[1] id=\"$array[2]\"/,/<\/$array[1]>/ d"
+done 
diff --git a/bin/osm/osm.pbf.update b/bin/osm/osm.pbf.update
new file mode 100755
index 0000000..6c6f445
--- /dev/null
+++ b/bin/osm/osm.pbf.update
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+
+GEOFABRIK_SERVER=http://download.geofabrik.de/asia/taiwan-updates
+PBF_FILE=$1
+
+# get latest sequence number
+echo Fetching the latest sequence number
+LATEST_SEQ=$(curl --silent http://download.geofabrik.de/asia/taiwan-updates/state.txt |\
+             tail -1 | cut -d'=' -f2)
+echo Latest sequence number is $LATEST_SEQ
+
+# get current sequence number
+SEQ=$(osmium fileinfo $PBF_FILE |\
+      grep osmosis_replication_sequence_number |\
+      cut -d'=' -f2)
+echo File sequence number is $SEQ
+
+# while server has osc file with given sequence number,
+# get it and do file update
+while
+    (( SEQ++ ))
+    [ $SEQ -le $LATEST_SEQ ]
+do
+    mkdir -p changes
+    SEQ_PATH=$(echo $SEQ | sed -r 's/(.{1})(.{3})/00\1\/\2/')
+    CHANGE_URL=$GEOFABRIK_SERVER/000/$SEQ_PATH.osc.gz
+    echo $CHANGE_URL
+    curl -o changes/$SEQ.osc.gz $CHANGE_URL && \
+    osmium apply-changes $PBF_FILE changes/$SEQ.osc.gz \
+        --output-header=osmosis_replication_sequence_number=$SEQ \
+        --overwrite \
+        --output $SEQ.osm.pbf
+
+    mv $PBF_FILE $((SEQ-1)).osm.pbf
+    mv $SEQ.osm.pbf $PBF_FILE
+done
+
+echo
+echo File sequence number is $((SEQ-1)), already the latest one on Geofrbrik
diff --git a/bin/osm/osm.query b/bin/osm/osm.query
new file mode 100755
index 0000000..8d0b9f7
--- /dev/null
+++ b/bin/osm/osm.query
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+osmium tags-filter - $@ --input-format=osm --output-format=osm --omit-referenced
diff --git a/bin/osm/sequence_number.sh b/bin/osm/sequence_number.sh
new file mode 100755
index 0000000..eb365d0
--- /dev/null
+++ b/bin/osm/sequence_number.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# $1 as --hour or --minute, $2 as timestamp
+# return the latest sequence number
+
+case $1 in
+    # hour difference with Tue Jun 4 03:00:00 UTC 2019
+    # sequence number=58940
+    --hour)
+    echo $[($2 - 1559617200)/3600 + 58940]
+    ;;
+
+    # minute difference with latest planet state file
+    --minute)
+    benchmark=benchmark
+    curl https://planet.openstreetmap.org/replication/minute/state > $benchmark
+    timeString=$(tail -1 $benchmark | tr -d 'timestamp=\\')
+    timestamp=$(date -d "$timeString" +%s)
+    seq=$(sed -n 2p $benchmark | tr -d "sequenceNumber=")
+    rm $benchmark
+    echo $[$seq - ($timestamp - $2)/60 - 1 ]
+esac