45 files changed, 855 insertions, 0 deletions

diff --git a/tools/check_upstream b/tools/check_upstream
new file mode 100755
index 0000000..3313e5b
--- /dev/null
+++ b/tools/check_upstream
@@ -0,0 +1,25 @@
+#! /bin/bash
+
+# This script is for repo forked from others
+# check $1(repo) if upstream branch origin/master is
+# ahead of local branch $2(default to dev)
+
+if [ ! -d "$1" ]; then
+    return 0
+fi
+
+head='dev'
+if [ $# -eq 2 ]
+then
+    head=$2
+fi
+
+cd "$1" && \
+git fetch origin && \
+if ! git rev-list "$head" | grep "$(git rev-parse origin/master)" > /dev/null
+then
+  [[ $(git pull my) == 'Alrady up to date.' ]] || \
+  echo "New commit at" "$1"
+fi
+
+echo "$(date)" check "$1" >> "$SETTING_DIR/log"
diff --git a/tools/csv.move_column b/tools/csv.move_column
new file mode 100755
index 0000000..a62701f
--- /dev/null
+++ b/tools/csv.move_column
@@ -0,0 +1,18 @@
+#! /bin/bash
+
+# show each field with index in csv
+echo -------------- > /dev/tty
+head -1 < $1 | sed 's/,/ /g' | awk '{for (i=1; i<=NF; i++) printf $i "_" i " "; print ""}' > /dev/tty
+echo -------------- > /dev/tty
+echo > /dev/tty
+
+# get index of lon/lat column
+read -p "Move which column? " origin_col
+read -p "To which index? " new_col
+
+cat $1 |
+# move lon and lat to the first and second column
+awk -F',' -v OFS="," -v origin_th=$origin_col -v new_th=$new_col '\
+    {for (i=1; i<= NF; i++) if (i == new_th) printf $origin_th OFS $i OFS; else if (i == origin_th); else printf $i OFS; print ""}\
+    ' |\
+sed 's/,$//g'
diff --git a/tools/csv.reorder b/tools/csv.reorder
new file mode 100755
index 0000000..8a64239
--- /dev/null
+++ b/tools/csv.reorder
@@ -0,0 +1,17 @@
+#! /bin/bash
+
+# show each field with index in csv
+echo -------------- > /dev/tty
+head -1 < $1 | awk -F',' '{for (i=1; i<=NF; i++) printf $i "_" i " "; print ""}' > /dev/tty
+echo -------------- > /dev/tty
+echo > /dev/tty
+
+read -p "type column numbers by new order, like 3 2 1: " order
+
+arrange=$(echo $order | sed -r 's/([^ ]+)/$\1/g' | tr ' ' ',')
+
+cat $1 |\
+awk -F',' "BEGIN{OFS=\",\"}{print $arrange}" |\
+tee /tmp/csv
+
+echo "Also copied to /tmp/csv" > /dev/tty
diff --git a/tools/csv2geojson b/tools/csv2geojson
new file mode 100755
index 0000000..028aed7
--- /dev/null
+++ b/tools/csv2geojson
@@ -0,0 +1,46 @@
+#! /bin/bash
+
+# -s to skip specify columns of longitude and latitude
+for i in "$@"
+do
+case $i in
+    -s)
+    lon_col=0; lat_col=1
+    shift;;
+
+    *)
+    csv=$i
+    shift;;
+esac
+done
+
+# if no -s option, just read from input
+if [ "$lon_col" != "0" ]; then
+    # show each field with index in csv
+    echo -------------- > /dev/tty
+    head -1 < $csv | awk -F',' '{for (i=1; i<=NF; i++) printf $i "_" i " "; print ""}' > /dev/tty
+    echo -------------- > /dev/tty
+    echo > /dev/tty
+
+    # get index of lon/lat column
+    read -p "Number of latitude column: " lat_col
+    read -p "Number of longitude column: " lon_col
+fi
+
+cat $csv |
+# move lon and lat to the first and second column
+awk -F',' -v lon_th=$lon_col -v lat_th=$lat_col '\
+    BEGIN{OFS=","}\
+    {printf $lon_th "," $lat_th; for (i=1; i<= NF; i++) if (i != lat_th && i != lon_th) printf "," $i; print ""}\
+    ' |\
+# change csv into array format, like [lon, lat, "field1", field2...]
+sed 's/[^,]*/"\0"/g; s/.*/[\0]/g' |\
+# wrap other fields as a json object, like [lon, lat, {...}]
+jq -s '.[0][2:] as $fields | .[1:][] | [.[0], .[1], ([$fields, .[2:]] | transpose | map({(.[0]): .[1]}) | add)]' |\
+# create array of geojson point features
+jq '{"type": "Feature",  "properties": .[2],  "geometry":{ "type": "Point",  "coordinates": [(.[0] | tonumber), (.[1] | tonumber)] } }' |\
+# wrap features as geojson format
+jq -s '{"type": "FeatureCollection", "features": .}' |\
+tee /tmp/geojson
+
+echo stored into /tmp/geojson > /dev/tty
diff --git a/tools/gpx/footer b/tools/gpx/footer
new file mode 100644
index 0000000..d0759c0
--- /dev/null
+++ b/tools/gpx/footer
@@ -0,0 +1 @@
+</gpx>
diff --git a/tools/gpx/gpx.check.py b/tools/gpx/gpx.check.py
new file mode 100755
index 0000000..10be97c
--- /dev/null
+++ b/tools/gpx/gpx.check.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import argparse
+import copy
+import fileinput
+from osgeo import ogr
+import osr
+import urllib.parse
+
+def rewrite_gpx(filename):
+    for line in fileinput.input(filename, inplace=True):
+        if fileinput.isfirstline() and "'" in line:
+            line = '<?xml version="1.0" encoding="UTF-8"?>'
+        if fileinput.filelineno() == 2 and "version" not in line:
+            line = line.replace('<gpx', '<gpx version="1.1"')
+        print(line.rstrip('\n'))
+
+def check_valid(filename, threshold, add_prefix):
+    rewrite_gpx(filename)
+
+    driver = ogr.GetDriverByName('GPX')
+    try:
+        dataSource = driver.Open(filename)
+    except Exception:
+        pass
+    if dataSource is None:
+        print("could not open")
+        sys.exit(1)
+
+    inSpatialRef = osr.SpatialReference()
+    inSpatialRef.ImportFromEPSG(4326)
+    outSpatialRef = osr.SpatialReference()
+    outSpatialRef.ImportFromEPSG(3857)
+    to3857 = osr.CoordinateTransformation(inSpatialRef, outSpatialRef)
+    to4326 = osr.CoordinateTransformation(outSpatialRef, inSpatialRef)
+
+    trkLayer = dataSource.GetLayer(4)
+    trkpt = trkLayer.GetNextFeature()
+    flag = False
+    while trkpt:
+        nextTrkpt = trkLayer.GetNextFeature()
+        if nextTrkpt:
+            geom1 = trkpt.GetGeometryRef()
+            geom1.Transform(to3857)
+            geom2 = nextTrkpt.GetGeometryRef()
+            geom2.Transform(to3857)
+            distance = geom1.Distance(geom2)
+
+            geom1.Transform(to4326)
+            geom2.Transform(to4326)
+            if distance >= threshold:
+                if not flag:
+                    print(f'{filename} has problem, the following urls shows the points with distance far from {threshold}m:')
+                    print()
+                    flag = True
+                    if add_prefix:
+                        dir = os.path.dirname(filename)
+                        if dir:
+                            dir += '/'
+                        os.rename(filename, f'{dir}invalid_{os.path.basename(filename)}')
+
+                geojson = '{{"type": "LineString", "coordinates": [[{}, {}], [{}, {}]]}}'.format(
+                    geom1.GetX(), geom1.GetY(),
+                    geom2.GetX(), geom2.GetY()
+                )
+                encoded = urllib.parse.quote(geojson)
+                print('http://geojson.io/#data=data:application/json,{}'.format(encoded))
+                print()
+        else:
+            break
+        trkpt = nextTrkpt
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('file', help="you can add multiple gpx files at the same time", nargs='+')
+    parser.add_argument("-i", help="add prefix to invalid files", action="store_true")
+    parser.add_argument("-d", help="distance of tolerance(m), 100 by default", dest="distance", default=100)
+    args = parser.parse_args()
+    for file in args.file:
+        check_valid(file, float(args.distance), args.i)
+
+if __name__ == '__main__':
+    main(sys.argv)
diff --git a/tools/gpx/gpx.merge_gpx.sh b/tools/gpx/gpx.merge_gpx.sh
new file mode 100755
index 0000000..4b024a7
--- /dev/null
+++ b/tools/gpx/gpx.merge_gpx.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+GPX_DIR=$(dirname $0)
+
+sed '/<trk/,/<\/trk>/ p' -nr | cat $GPX_DIR/header - $GPX_DIR/footer
diff --git a/tools/gpx/gpx.merge_trk.sh b/tools/gpx/gpx.merge_trk.sh
new file mode 100755
index 0000000..c3a72d4
--- /dev/null
+++ b/tools/gpx/gpx.merge_trk.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+sed '/<trk>/,/<\/name>/ d; /<\/trk>/ d; /<\/gpx>/ i \ \ <\/trk>' |\
+awk '/<trkseg>/ && !x {print "  <trk>\n    <name>combined_trk</name>"; x=1} 1'
diff --git a/tools/gpx/header b/tools/gpx/header
new file mode 100644
index 0000000..1912e8b
--- /dev/null
+++ b/tools/gpx/header
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no" ?>
+<gpx xmlns="http://www.topografix.com/GPX/1/1" creator="MapSource 6.10.2" version="1.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd">
+
+  <metadata>
+    <link href="http://www.garmin.com">
+      <text>Garmin International</text>
+    </link>
+    <time>2019-12-18T06:17:16Z</time>
+    <bounds maxlat="24.554773" maxlon="121.293261" minlat="24.534487" minlon="121.284309"/>
+  </metadata>
diff --git a/tools/load-settings.sh b/tools/load-settings.sh
new file mode 100755
index 0000000..30a29b0
--- /dev/null
+++ b/tools/load-settings.sh
@@ -0,0 +1,24 @@
+if [[ -z "$SETTING_DIR" ]]; then
+    SETTING_DIR=$HOME/settings
+fi
+
+# load custom aliases
+source $SETTING_DIR/alias
+
+# Add custom scripts into PATH
+PATH=$PATH:$SETTING_DIR/tools
+find $SETTING_DIR/tools -type d | sed 1d |\
+while read dir; do
+    PATH=$PATH:$dir
+done
+
+# sync with important git repos
+$SETTING_DIR/tools/sync.sh
+
+# load custom functions
+OSM_UTIL_DIR=$SETTING_DIR/tools/osm
+source $OSM_UTIL_DIR/osm
+
+# go
+PATH=$PATH:$HOME/go/bin
+
diff --git a/tools/match-road.sh b/tools/match-road.sh
new file mode 100755
index 0000000..d9beaae
--- /dev/null
+++ b/tools/match-road.sh
@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+#
+# Author: Pham
+#
+# This script accepts a single GPX file as parameter and
+# output the processed GPX body to STDOUT, using Mapbox Map Matching API v4.
+# read doc at: https://docs.mapbox.com/api/legacy/map-matching-v4/
+#
+# Example:
+#
+# match-road.sh raw.gpx > new.gpx
+#
+# Hint:
+#
+# Remember to put Mapbox Access Token at the top!
+
+#set -x
+set -e
+
+# put yout Mapbox token here
+ACCESS_TOKEN=$(cat ~/settings/tokens/mapbox)
+# number of coordinates for each Mapbox Map Matching API request, Maximum value is 100
+LIMIT=100
+# define the lowest confidence of accepted matched points
+THRESHOLD=0.0001
+
+if [[ -z $1 ]]; then echo "You need to give a gpx file!"; exit 1; fi
+ORIGIN_DATA=/tmp/$(basename $1).origin
+RESPONSES=/tmp/$(basename $1).responses && true > $RESPONSES
+MATCHED=/tmp/$(basename $1).matched
+
+# extract data from the given gpx file
+# only keep first point and remove the rest which in the same "second"
+# input:  [gpx format]
+# output: [121.0179739,14.5515336] 1984-01-01T08:00:46
+get_data() {
+    sed -nr '/<trkpt /, /<\/trkpt>/ {H; /<\/trkpt>/ {x; s/\n/ /g; p; s/.*//; x}}' $1 |
+    sed -nr 'h; s/.*lon="([^"]+).*/\1/; H; g
+                s/.*lat="([^"]+).*/\1/; H; g
+                # If trkpt has no time, leave it blank
+                /time/ {
+                    s/.*<time>([^.]+).*<\/time>.*/\1/
+                    H; g
+                }
+                s/^[^\n]+\n//; s/\n/ /g; p' |
+    sed -r 's/^([^ ]+) ([^ ]+)/[\1,\2]/' |
+    awk '!_[$2]++'
+}
+
+# Output GeoJSON object for Map Matching API
+# input:  [121.0179739,14.5515336] 1984-01-01T08:00:46
+# output: {type: "Feature", properties: {coordTimes: [...]}, geometry: {type: "LineString", coordinates: [...]}}
+make_geojson() {
+    # change input to format like: [[lon, lat], time]
+    awk '{printf("[%s,\"%s\"]\n", $1, $2)}' |
+    jq '[inputs] | {type: "Feature", geometry: {type: "LineString", coordinates: map(.[0])}, properties: {coordTimes: (map(.[1]))}}'
+}
+
+# Read GeoJSON body from STDIN, and return result from Mapbox Map Matching API
+query_matched_points() {
+    curl -X POST -s --data @- \
+        --header "Content-Type:application/json" \
+        https://api.mapbox.com/matching/v4/mapbox.driving.json?access_token=$ACCESS_TOKEN
+}
+
+# Get valid data from Map Matching API response
+# output format is [coordinates] [index-of-original-data], like:
+# [121.0179739,14.5515336] 35
+# If the point is newly added, the index would be -1, like
+# [121.0189339,14.5525931] -1
+validate_matched_points() {
+    VALID_DATA=$(jq ".features[] | if(.properties.confidence < $THRESHOLD) then .geometry.coordinates=(.properties.indices|map(.+1)) else . end")
+
+    echo $VALID_DATA |
+    jq -cr '.properties | [.matchedPoints, (.indices | map(.+1))] | transpose[] | "\(.[0]) \(.[1])"' > $MATCHED
+
+    echo $VALID_DATA | jq -c '.geometry.coordinates[]' |
+    while read point; do
+        if [[ ${point:0:1} != '[' ]]; then
+            echo $(sed -n "$point p" $ORIGIN_DATA) | while read coor time; do echo $coor $(date -d $time +%s); done
+            sed -i 1d $MATCHED
+        elif head -1 $MATCHED | grep -F $point > /dev/null; then
+            index=$(head -1 $MATCHED | cut -d' ' -f2)
+            echo $point $(sed -n "$index p" $ORIGIN_DATA | cut -d' ' -f2 | date -f - +%s)
+            sed -i 1d $MATCHED
+        else
+            echo $point -1
+        fi
+    done
+}
+
+# Put existing timestamps to matched points, and interpolate new timestamps into new points
+complete_data() {
+    # interpolate timestamps to newly added points
+    awk '{COOR[NR][0]=$1; N++; COOR[NR][1]=$2} END{for (i=1; i<=N; i++) {printf COOR[i][0]; if (COOR[i][1] != -1) {print " "COOR[i][1]; LAST=i} else {while(COOR[i+n][1] == -1) n++; print " "COOR[LAST][1]+(COOR[i+n][1]-COOR[LAST][1])*(i-LAST)/(i+n-LAST)}}}' |
+    while read coor unix_time; do
+        # Transform unix timestamp into human readable time format, like following:
+        # Transform [121.018088,14.5516] 18.50
+        # Into      [121.018088,14.5516] 1970-01-01T08:00:18.50Z
+        echo $coor $(date -d @$unix_time +'%Y-%m-%dT%H:%M:%S.%2NZ')
+    done
+}
+
+# Make GPX format for output
+# Take input with format: [lon,lat] [time]
+make_gpx() {
+    sed -E 's/\[([^,]+),([^,]+)\] (.*)/      <trkpt lon="\1" lat="\2"><time>\3<\/time><\/trkpt>/' |
+    sed "1i \
+<gpx version=\"1.1\" creator=\"Garmin Connect\"\n\
+  xsi:schemaLocation=\"http://www.topografix.com/GPX/1/1 http://www.topografix.com/GPX/1/1/gpx.xsd http://www.garmin.com/xmlschemas/GpxExtensions/v3 http://www.garmin.com/xmlschemas/GpxExtensionsv3.xsd http://www.garmin.com/xmlschemas/TrackPointExtension/v1 http://www.garmin.com/xmlschemas/TrackPointExtensionv1.xsd\"\n\
+  xmlns=\"http://www.topografix.com/GPX/1/1\"\n\
+  xmlns:gpxtpx=\"http://www.garmin.com/xmlschemas/TrackPointExtension/v1\"\n\
+  xmlns:gpxx=\"http://www.garmin.com/xmlschemas/GpxExtensions/v3\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n\
+  <trk>\n    <trkseg>" |
+    sed "\$a \ \ \ \ <\/trkseg>\n  <\/trk>\n<\/gpx>"
+}
+
+get_data $1 > $ORIGIN_DATA
+
+#RAW_REQUEST=$(basename $1 | tr '.' '_')_request.geojson
+#cat $ORIGIN_DATA | make_geojson | jq '.properties.stroke="#ff0000"' > $RAW_REQUEST
+
+# Consume raw data with serveral request
+while [ -s $ORIGIN_DATA ]; do
+    # Take original data by limited points for each time: [121.0179739,14.5515336] 1984-01-01T08:00:46
+    # Make geojson body: {type: "Feature", properties: {coordTimes: [...]}, geometry: {type: "LineString", coordinates: [...]}}
+    # Call Mapbox Map Matching API: {geometry: {coordinates: [...]}, properties: {confidence: 0.9, matching: [...], indices: [...]}}
+    # Get valid point and index by confidence value: [121.0179739,14.5515336] 5
+    # complete_data
+    head -$LIMIT $ORIGIN_DATA |
+    make_geojson |
+    query_matched_points |
+    tee -a $RESPONSES |
+    validate_matched_points |
+    complete_data
+
+    # Remove processed raw data
+    sed -i "1,$LIMIT d" $ORIGIN_DATA
+done | make_gpx
+#make_geojson > test.geojson
+
+#RAW_RESPONSE=$(basename $1 | tr '.' '_')_response.geojson
+#MATCHED_POINTS=$(basename $1 | tr '.' '_')_matched.geojson
+#jq . $RESPONSES | jq -s '.[0].features=[.[]|.features[]] | .[0] | del(.code) | .features=(.features|map(.properties.stroke="#00ff00"))' > $RAW_RESPONSE
+#jq ".features=(.features|map(select(.properties.confidence>=$THRESHOLD).geometry.coordinates=.properties.matchedPoints)|map(.properties.stroke=\"#0000ff\"))" $RAW_RESPONSE > $MATCHED_POINTS
+#
+#DEBUG=$(basename $1 | tr '.' '_')_total.geojson
+#cat $RAW_REQUEST $RAW_RESPONSE $MATCHED_POINTS | jq -s '{type: "FeatureCollection", features: ([.[0]] + .[1].features + .[2].features)}' > $DEBUG
diff --git a/tools/mvt_decode.py b/tools/mvt_decode.py
new file mode 100644
index 0000000..0f661dd
--- /dev/null
+++ b/tools/mvt_decode.py
@@ -0,0 +1,11 @@
+import mapbox_vector_tile
+import sys
+
+#Python3
+
+mvt = sys.argv[1]
+
+with open(mvt, 'rb') as f:
+    data = f.read()
+    decoded_data = mapbox_vector_tile.decode(data)
+    print(decoded_data)
diff --git a/tools/osm/josm_install.sh b/tools/osm/josm_install.sh
new file mode 100755
index 0000000..520875a
--- /dev/null
+++ b/tools/osm/josm_install.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# scripts from https://josm.openstreetmap.de/wiki/Download#Webstart
+echo deb https://josm.openstreetmap.de/apt $(lsb_release -sc) universe | sudo tee /etc/apt/sources.list.d/josm.list > /dev/null &&\
+wget -q https://josm.openstreetmap.de/josm-apt.key -O- | sudo apt-key add - &&\
+sudo apt-get update &&\
+sudo apt-get remove josm josm-plugins &&\
+sudo apt-get install josm
diff --git a/tools/osm/osm b/tools/osm/osm
new file mode 100644
index 0000000..a595fb7
--- /dev/null
+++ b/tools/osm/osm
@@ -0,0 +1,17 @@
+#! /bin/sh
+
+#export OSM_SERVER=https://master.apis.dev.openstreetmap.org
+export OSM_SERVER=https://api.openstreetmap.org
+
+export OSM_API=$OSM_SERVER/api/0.6
+export OSM_USER_PASSWD=$(cat $SETTING_DIR/tokens/osm 2>/dev/null)
+
+FILENAME=$0
+
+utils.osm() {
+    vim $FILENAME && source $FILENAME
+}
+
+osm.api() {
+    curl -X GET $OSM_API/$1/$2
+}
diff --git a/tools/osm/osm.api.changeset.add b/tools/osm/osm.api.changeset.add
new file mode 100755
index 0000000..3e4878e
--- /dev/null
+++ b/tools/osm/osm.api.changeset.add
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+element=$(cat)
+header=$(echo $element | grep -E "<(node|way|relation)\s")
+ele_type=$(echo $header | sed -r 's/.*<(node|way|relation).*$/\1/')
+id=$(echo $header | sed -r 's/.* id=\"([^"]+)\".*$/\1/')
+
+echo $element | \
+sed -r "s/^( *<(node|way|relation).*version[^ ]+ )(.*)$/\1changeset=\"$1\">/" | \
+curl -X PUT -u $OSM_USER_PASSWD -i -T - $OSM_API/$ele_type/$id
diff --git a/tools/osm/osm.api.changeset.close b/tools/osm/osm.api.changeset.close
new file mode 100755
index 0000000..3b016c3
--- /dev/null
+++ b/tools/osm/osm.api.changeset.close
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+curl -X PUT -u $OSM_USER_PASSWD -i $OSM_API/changeset/$1/close
diff --git a/tools/osm/osm.api.changeset.create b/tools/osm/osm.api.changeset.create
new file mode 100755
index 0000000..ed2601a
--- /dev/null
+++ b/tools/osm/osm.api.changeset.create
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+echo -n "type comment: "
+read comment
+
+info="<osm>
+        <changeset>
+          <tag k='comment' v='$comment'/>
+          <tag k='created_by' v='bash script'/>
+          <tag k='bot' v='yes'/>
+        </changeset>
+      </osm>
+     "
+
+changeset_id=$(echo $info |\
+               curl -u $OSM_USER_PASSWD --upload-file - $OSM_API/changeset/create |\
+               tail -1)
+
+echo
+echo "changeset created, check $OSM_SERVER/changeset/$changeset_id"
+echo "$changeset_id is copied into clipboard"
diff --git a/tools/osm/osm.api.changeset.update b/tools/osm/osm.api.changeset.update
new file mode 100755
index 0000000..7b32741
--- /dev/null
+++ b/tools/osm/osm.api.changeset.update
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+echo -n 'type comment: '
+read -r comment
+
+echo "<osm><changeset><tag k=\"comment\" v=\"$comment\"/></changeset></osm>" | \
+curl -X PUT -u $OSM_USER_PASSWD -i -T - $OSM_API/changeset/$1
diff --git a/tools/osm/osm.api.changeset.upload b/tools/osm/osm.api.changeset.upload
new file mode 100755
index 0000000..96bdff4
--- /dev/null
+++ b/tools/osm/osm.api.changeset.upload
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+cat $2 |\
+sed -r "/<(node|way|relation)/ s/>/ changeset=\"$1\">/" |\
+tee /dev/tty |\
+curl -X POST -u $OSM_USER_PASSWD -i -T - $OSM_API/changeset/$1/upload
diff --git a/tools/osm/osm.api.fetch b/tools/osm/osm.api.fetch
new file mode 100755
index 0000000..8460c5f
--- /dev/null
+++ b/tools/osm/osm.api.fetch
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+# get .osm format data
+curl -X GET $OSM_API/$1/$2 |\
+tee /tmp/osm &&\
+echo content of $1 $2 is copied into /tmp/osm > /dev/tty
diff --git a/tools/osm/osm.api.fetch.full b/tools/osm/osm.api.fetch.full
new file mode 100755
index 0000000..5ee78b8
--- /dev/null
+++ b/tools/osm/osm.api.fetch.full
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+curl -X GET $OSM_API/$1/$2/full |\
+tee /tmp/osm &&\
+echo "\n" content of $1 $2 and its members are copied into /tmp/osm > /dev/tty
diff --git a/tools/osm/osm.api.fetch.history b/tools/osm/osm.api.fetch.history
new file mode 100755
index 0000000..bc837bd
--- /dev/null
+++ b/tools/osm/osm.api.fetch.history
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+curl -X GET $OSM_API/$1/$2/history |\
+tee /tmp/osm &&\
+echo &&\
+echo "\n" history of $1 $2 are copied into /tmp/osm > /dev/tty
diff --git a/tools/osm/osm.api.member.relation b/tools/osm/osm.api.member.relation
new file mode 100755
index 0000000..5addef7
--- /dev/null
+++ b/tools/osm/osm.api.member.relation
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+osm.api.referenced.relation $1 $2 |\
+osm.member.relation $1 $2 $3
diff --git a/tools/osm/osm.api.referenced.relation b/tools/osm/osm.api.referenced.relation
new file mode 100755
index 0000000..2ce0631
--- /dev/null
+++ b/tools/osm/osm.api.referenced.relation
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+curl -X GET $OSM_API/$1/$2/relations |\
+tee /tmp/osm &&\
+echo &&\
+echo "\n" relations contain $1 $2 are copied into /tmp/osm > /dev/tty
diff --git a/tools/osm/osm.api.referenced.way b/tools/osm/osm.api.referenced.way
new file mode 100755
index 0000000..1d84238
--- /dev/null
+++ b/tools/osm/osm.api.referenced.way
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+curl -X GET $OSM_API/node/$1/ways |\
+tee /tmp/osm &&\
+echo ways contain node $1 are copied into /tmp/osm > /dev/tty
diff --git a/tools/osm/osm.api.upload.to b/tools/osm/osm.api.upload.to
new file mode 100755
index 0000000..3979220
--- /dev/null
+++ b/tools/osm/osm.api.upload.to
@@ -0,0 +1,12 @@
+#! /bin/sh
+
+# allows multiple elements in osm body
+tee /tmp/osm |\
+osm.list.ids |\
+sed 's#.*#osm.extract \0 < /tmp/osm#g' |\
+sed "s/.*/\0 \| osm.api.changeset.add $1/g" |\
+while read -r command
+do
+    echo $command
+    source<(echo "($command &)")
+done
diff --git a/tools/osm/osm.file.get b/tools/osm/osm.file.get
new file mode 100755
index 0000000..e2b67b4
--- /dev/null
+++ b/tools/osm/osm.file.get
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+file=$1; shift
+osmium getid $file $@ --output-format=osm
diff --git a/tools/osm/osm.file.get.full b/tools/osm/osm.file.get.full
new file mode 100755
index 0000000..50ee2f1
--- /dev/null
+++ b/tools/osm/osm.file.get.full
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+file=$1; shift
+osmium getid $file $@ --output-format=osm --add-referenced
diff --git a/tools/osm/osm.file.query b/tools/osm/osm.file.query
new file mode 100755
index 0000000..4a01b5b
--- /dev/null
+++ b/tools/osm/osm.file.query
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+file=$1; shift
+osmium tags-filter $file $@ --output-format=osm --omit-referenced
diff --git a/tools/osm/osm.goto b/tools/osm/osm.goto
new file mode 100755
index 0000000..25bceaa
--- /dev/null
+++ b/tools/osm/osm.goto
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+xdg-open https://www.openstreetmap.org/$1/$2
diff --git a/tools/osm/osm.help b/tools/osm/osm.help
new file mode 100755
index 0000000..aa3b36b
--- /dev/null
+++ b/tools/osm/osm.help
@@ -0,0 +1,34 @@
+#! /bin/sh
+
+    echo '
+COMMANDS:
+  osm.help
+  util.osm.edit
+
+  osm.api.changeset.create     create a new changeset and return its ID
+  osm.api.changeset.update     [changeset ID] update changeset with new comment
+  osm.api.changeset.upload     [changeset ID] [osc file] upload osc file to the given changeset
+  osm.api.changeset.add        [changeset ID] STDIN=osm element, add it into changeset
+  osm.api.changeset.close      [changeset ID] close the given changeset
+  osm.api.fetch                [element type] [element ID]
+  osm.api.fetch.full           like .fetch, but return with all referenced elements
+  osm.api.fetch.history        [element type] [element ID]
+  osm.api.member.relation      [element type] [element ID] [member type] get first relations reference it as member type
+  osm.api.referenced.relation  [element type] [element ID] get all relations reference this element
+  osm.api.referenced.way       [element type] [element ID] get all ways reference this element
+  osm.api.upload.to
+
+  osm.file.cat
+  osm.file.get
+  osm.file.get.full
+  osm.file.query
+  osm.list.ids
+  osm.list.tag
+  osm.list.tags
+
+  osm.get.id
+  osm.query
+  osm.member.relation
+  osm.pbf.update
+  osm.osm.update
+    '
diff --git a/tools/osm/osm.list.ids b/tools/osm/osm.list.ids
new file mode 100755
index 0000000..8e5d870
--- /dev/null
+++ b/tools/osm/osm.list.ids
@@ -0,0 +1,3 @@
+#! /bin/sh
+
+sed -nr 's/.*<(node|way|relation) id=\"([^"]+)\".*/\1 \2/p'
diff --git a/tools/osm/osm.list.tag b/tools/osm/osm.list.tag
new file mode 100755
index 0000000..76649a0
--- /dev/null
+++ b/tools/osm/osm.list.tag
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+ele_pattern="(node|way|relation)"
+sed -nr "/<$ele_pattern/,/<\/$ele_pattern/ {
+    /<tag k=\"$1\"/ {
+        s/.*v=\"([^\"]+)\".*/\1/
+        h
+    }
+    /<\/$ele_pattern/ {x;p;s/.*//;x}
+}"
diff --git a/tools/osm/osm.list.tags b/tools/osm/osm.list.tags
new file mode 100755
index 0000000..767d32a
--- /dev/null
+++ b/tools/osm/osm.list.tags
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+content=$(cat)
+echo $content | osm.list.ids | tr ' ' ',' > /tmp/osm
+
+for tag in $@
+do
+    echo $content |\
+    osm.list.tag $tag |\
+    paste -d',' /tmp/osm - > /tmp/osm.new &&\
+    mv /tmp/osm.new /tmp/osm
+done
+
+cat /tmp/osm
+echo "\ntag list is also copied into /tmp/osm" > /dev/tty
diff --git a/tools/osm/osm.member.relation b/tools/osm/osm.member.relation
new file mode 100755
index 0000000..f8d4598
--- /dev/null
+++ b/tools/osm/osm.member.relation
@@ -0,0 +1,8 @@
+#! /bin/sh
+
+sed -nr "/<relation/,/<\/relation/ {
+    /<relation/ {s/.* id=\"([^\"]+)\".*/\1/;h}
+    /<member type=\"$1\" ref=\"$2\" role=\"$3\"/ {g;p;q}
+}
+$ {s/.*//;P}
+" | head -1
diff --git a/tools/osm/osm.osc.by_member b/tools/osm/osm.osc.by_member
new file mode 100755
index 0000000..5f9d9be
--- /dev/null
+++ b/tools/osm/osm.osc.by_member
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+while read -r line
+do
+
+    TYPE=$(echo $line | cut -d ' ' -f1) # field1 is type
+    ID=$(echo $line | cut -d ' ' -f2)   # field2 is ID
+
+    NEW_MEMBERS=$(echo $line |\
+               cut -d' ' -f3- |\
+               sed -r 's/([0-9]+)/<member type=\"relation\" ref=\"\1\" role=\"subarea\"\/>/g')
+
+    echo $NEW_MEMBERS
+
+    # print matched element with new tags to .osc file
+    cat $1 |\
+    sed -nr "/<$TYPE id=\"$ID\"/,/<\/$TYPE/ {
+                 /<$TYPE id=\"$ID\"/ {
+                     s/(version=\"[0-9]+\")(.*)/\1>/
+                     a \ \ \ \ $NEW_MEMBERS
+                 }
+                 p
+                 /<\/$TYPE/ q
+            }" >> $1.osc
+done
+
+# Add .osc structure for output
+sed -ir '1 i <osmChange version="0.6" generator="bash script">
+         1 i <modify>
+         $ a </modify>
+         $ a </osmChange>' $1.osc
diff --git a/tools/osm/osm.osc.by_tag b/tools/osm/osm.osc.by_tag
new file mode 100755
index 0000000..cc92c1d
--- /dev/null
+++ b/tools/osm/osm.osc.by_tag
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# create new tags from input line, for example: 
+# field1         field2       field3     field4   field5     field6   field7       field8...
+# [element type] [element ID] key1_added "value1" key2_added "value2" key3_removed key4_removed...
+
+# key should not quoted, value must be quoted
+# And keys which need to be removed must be placed at the end
+while read -r line
+do
+    TYPE=$(echo $line | cut -d ' ' -f1) # field1 is type
+    ID=$(echo $line | cut -d ' ' -f2)   # field2 is ID
+
+    # transform key-value pair into tag format:
+    # <tag k="[key]" v="[value]"/>
+    # keys without values are omitted
+    NEW_TAGS=$(echo $line |\
+               cut -d' ' -f3- |\
+               sed -r 's/([^ "]+) (\"[^"]+\")/<tag k=\"\1\" v=\2\/>/g; s/>[^"]*$/>/')
+
+    # get regex pattern need to removed from original osm element:
+    # key1|key2|key3|key4
+    TAG_PATTERN=$(echo $line |\
+                  cut -d' ' -f3- | xargs -n2 echo |\
+                  cut -d' ' -f1 | paste -s -d'|')
+
+    echo $NEW_TAGS > /dev/tty
+
+    # print matched element with new tags to .osc file
+    cat $1 |\
+    sed -nr "/<$TYPE id=\"$ID\"/,/<\/$TYPE/ {
+                 /<$TYPE id=\"$ID\"/ {
+                     s/(version=\"[0-9]+\")(.*)/\1>/
+                     a \ \ \ \ $NEW_TAGS
+                 }
+                 /<tag k=\"($TAG_PATTERN)\"/ !p
+                 /<\/$TYPE/ q
+            }" >> $1.osc
+done
+
+# Add .osc structure for output
+sed -ir '1 i <osmChange version="0.6" generator="bash script">
+         1 i <modify>
+         $ a </modify>
+         $ a </osmChange>' $1.osc
diff --git a/tools/osm/osm.osm.remove b/tools/osm/osm.osm.remove
new file mode 100755
index 0000000..83d6ea8
--- /dev/null
+++ b/tools/osm/osm.osm.remove
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+while read -r line
+do
+    # put element type and element ID into array
+    array=( $(echo $line) )
+    cat $1 |\
+    sed -i "/<$array[1] id=\"$array[2]\"/,/<\/$array[1]>/ d"
+done 
diff --git a/tools/osm/osm.pbf.update b/tools/osm/osm.pbf.update
new file mode 100755
index 0000000..cd115a7
--- /dev/null
+++ b/tools/osm/osm.pbf.update
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+GEOFABRICK_SERVER=http://download.geofabrik.de/asia/taiwan-updates
+PBF_FILE=$1
+
+# get latest sequence number
+echo Fetching the latest sequence number
+LATEST_SEQ=$(curl --silent http://download.geofabrik.de/asia/taiwan-updates/state.txt |\
+             tail -1 | cut -d'=' -f2)
+echo Latest sequence number is $LATEST_SEQ
+
+# get current sequence number
+SEQ=$(osmium fileinfo $PBF_FILE |\
+      grep osmosis_replication_sequence_number |\
+      cut -d'=' -f2)
+echo File sequence number is $SEQ
+
+# while server has osc file with given sequence number,
+# get it and do file update
+while
+    (( SEQ++ ))
+    [ $SEQ -le $LATEST_SEQ ]
+do
+    mkdir -p changes
+    SEQ_PATH=$(echo $SEQ | sed -r 's/(.{1})(.{3})/00\1\/\2/')
+    CHANGE_URL=$GEOFABRICK_SERVER/000/$SEQ_PATH.osc.gz
+    echo $CHANGE_URL
+    curl -o changes/$SEQ.osc.gz $CHANGE_URL && \
+    osmium apply-changes $PBF_FILE changes/$SEQ.osc.gz \
+        --output-header=osmosis_replication_sequence_number=$SEQ \
+        --overwrite \
+        --output $SEQ.osm.pbf
+
+    mv $PBF_FILE $((SEQ-1)).osm.pbf
+    mv $SEQ.osm.pbf $PBF_FILE
+done
+
+echo
+echo File sequence number is $((SEQ-1)), already the latest one on Geofrbrik
diff --git a/tools/osm/osm.query b/tools/osm/osm.query
new file mode 100755
index 0000000..8d0b9f7
--- /dev/null
+++ b/tools/osm/osm.query
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+osmium tags-filter - $@ --input-format=osm --output-format=osm --omit-referenced
diff --git a/tools/osm/sequence_number.sh b/tools/osm/sequence_number.sh
new file mode 100755
index 0000000..eb365d0
--- /dev/null
+++ b/tools/osm/sequence_number.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# $1 as --hour or --minute, $2 as timestamp
+# return the latest sequence number
+
+case $1 in
+    # hour difference with Tue Jun 4 03:00:00 UTC 2019
+    # sequence number=58940
+    --hour)
+    echo $[($2 - 1559617200)/3600 + 58940]
+    ;;
+
+    # minute difference with latest planet state file
+    --minute)
+    benchmark=benchmark
+    curl https://planet.openstreetmap.org/replication/minute/state > $benchmark
+    timeString=$(tail -1 $benchmark | tr -d 'timestamp=\\')
+    timestamp=$(date -d "$timeString" +%s)
+    seq=$(sed -n 2p $benchmark | tr -d "sequenceNumber=")
+    rm $benchmark
+    echo $[$seq - ($timestamp - $2)/60 - 1 ]
+esac
diff --git a/tools/refresh-todo.sh b/tools/refresh-todo.sh
new file mode 100755
index 0000000..7931536
--- /dev/null
+++ b/tools/refresh-todo.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# $1 as file, $2 as topic (Daily, Weekly, Monthly)
+# change markdown check-list to empty
+sed -i "/^## $2/,/^$/ s/\[.\]/\[ \]/" $1
diff --git a/tools/sync.sh b/tools/sync.sh
new file mode 100755
index 0000000..1c33138
--- /dev/null
+++ b/tools/sync.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+cd $SETTING_DIR && git pull --quiet || echo in `pwd` > /dev/tty &
+cd ~/vimwiki && git pull --quiet || echo in `pwd` > /dev/tty &
+check_upstream ~/git/tig || echo in `pwd` > /dev/tty &
+check_upstream ~/.vim_runtime || echo in `pwd` > /dev/tty &
diff --git a/tools/upload-github-release-asset.sh b/tools/upload-github-release-asset.sh
new file mode 100755
index 0000000..0325530
--- /dev/null
+++ b/tools/upload-github-release-asset.sh
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# Author: Stefan Buck
+# License: MIT
+# https://gist.github.com/stefanbuck/ce788fee19ab6eb0b4447a85fc99f447
+#
+#
+# This script accepts the following parameters:
+#
+# * owner
+# * repo
+# * tag
+# * filename
+# * github_api_token
+# * overwrite (optional, could be ture, false, delete, default to be false)
+#
+# Script to upload a release asset using the GitHub API v3.
+#
+# Example:
+#
+# upload-github-release-asset.sh github_api_token=TOKEN owner=stefanbuck repo=playground tag=v0.1.0 filename=./build.zip overwrite=true
+#
+
+# Check dependencies.
+set -e
+xargs=$(which gxargs || which xargs)
+
+# Validate settings.
+[ "$TRACE" ] && set -x
+
+CONFIG=$@
+
+for line in $CONFIG; do
+  eval "$line"
+done
+
+# Define variables.
+GH_API="https://api.github.com"
+GH_REPO="$GH_API/repos/$owner/$repo"
+GH_TAGS="$GH_REPO/releases/tags/$tag"
+AUTH="Authorization: token $github_api_token"
+WGET_ARGS="--content-disposition --auth-no-challenge --no-cookie"
+CURL_ARGS="-LJO#"
+
+if [[ "$tag" == 'LATEST' ]]; then
+  GH_TAGS="$GH_REPO/releases/latest"
+fi
+
+# Validate token.
+curl -o /dev/null -sH "$AUTH" $GH_REPO || { echo "Error: Invalid repo, token or network issue!";  exit 1; }
+
+# Read asset tags.
+response=$(curl -sH "$AUTH" $GH_TAGS)
+
+# Get ID of the release.
+eval $(echo "$response" | grep -m 1 "id.:" | grep -w id | tr : = | tr -cd '[[:alnum:]]=' | sed 's/id/release_id/')
+[ "$release_id"  ] || { echo "Error: Failed to get release id for tag: $tag"; echo "$response" | awk 'length($0)<100' >&2; exit 1; }
+
+# Get ID of the asset based on given filename.
+# If exists, delete it.
+eval $(echo "$response" | grep -C2 "\"name\":.\+$(basename $filename)" | grep -m 1 "id.:" | grep -w id | tr : = | tr -cd '[[:alnum:]]=' | sed 's/id/asset_id/')
+if [ "$asset_id" = ""  ]; then
+    echo "No need to overwrite asset"
+else
+    if [[ "$overwrite" == "true" ]] || [[ "$overwrite" == "delete" ]]; then
+        echo "Deleting asset($asset_id)... "
+        curl  -X "DELETE" -H "Authorization: token $github_api_token" "https://api.github.com/repos/$owner/$repo/releases/assets/$asset_id"
+        if [[ "$overwrite" == "delete" ]]; then
+            exit 0
+        fi
+    else
+        echo "File already exists on tag $tag"
+        echo "If you want to overwrite it, set overwrite=true"
+        exit 1
+    fi
+fi
+
+# Upload asset
+echo "Uploading asset... "
+
+# Construct url
+GH_ASSET="https://uploads.github.com/repos/$owner/$repo/releases/$release_id/assets?name=$(basename $filename)"
+
+curl --data-binary @"$filename" -H "Authorization: token $github_api_token" -H "Content-Type: application/octet-stream" $GH_ASSET
diff --git a/tools/yaoren.sh b/tools/yaoren.sh
new file mode 100755
index 0000000..58d95df
--- /dev/null
+++ b/tools/yaoren.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+curl http://blog.sina.com.cn/u/1281912467 |\
+sed -nr "/第二列start/{
+
+}"
+#       <div class="SG_connBody">
+#                                                                                                                       <div class="bloglist">
+#                                                                                       <div class="blog_title_h">
+#                                                       <span class="img1"></span>
+#                                                                                                               <div id="t_10001_4c686e930102z2ft" class="blog_title">
+#                                                               <a href="http://blog.sina.com.cn/s/blog_4c686e930102z2ft.html" target="_blank">咬人画的-秋之菜饼</a>
+
+#
+
+sed -nr "s/.*real_src =\"(.*\/\/[^/]+\/)[^/]+(\/.*)&amp.*/\1orignal\2/p" tmp > jojo