30 files changed, 290 insertions, 1 deletions

diff --git a/scripts/csv.move_column b/scripts/csv/csv.move_column
index a62701f..a62701f 100755
--- a/scripts/csv.move_column
+++ b/scripts/csv/csv.move_column
diff --git a/scripts/csv.reorder b/scripts/csv/csv.reorder
index ef9766b..df928f1 100755
--- a/scripts/csv.reorder
+++ b/scripts/csv/csv.reorder
@@ -12,6 +12,8 @@ read -p "type column numbers by new order, like 3 2 1: " order
 arrange=$(echo $order | sed -r 's/([^ ]+)/$\1/g' | tr ' ' ',')
 
 cat $1 |\
-awk -F',' "BEGIN{OFS=\",\"}{print $arrange}"
+awk -F',' "BEGIN{OFS=\",\"}{print $arrange}" |\
+tee /tmp/csv |\
+echo "Also copied to /tmp/csv" > /dev/tty
 
 
diff --git a/scripts/csv2geojson b/scripts/csv/csv2geojson
index 085e12e..085e12e 100755
--- a/scripts/csv2geojson
+++ b/scripts/csv/csv2geojson
diff --git a/scripts/osm/osm.api.change.create b/scripts/osm/osm.api.change.create
new file mode 100644
index 0000000..1f17334
--- /dev/null
+++ b/scripts/osm/osm.api.change.create
@@ -0,0 +1,21 @@
+#!/bin/sh
+
+echo -n "type comment: "
+read comment
+
+info="<osm>
+        <changeset>
+          <tag k='comment' v='$comment'/>
+          <tag k='bot' v='yes'/>
+        </changeset>
+      </osm>
+     "
+
+echo $info |\
+curl -u $OSM_USER_PASSWD --upload-file - $OSM_API/changeset/create |\
+tee /dev/tty |\
+tail -1 | read changeset_id
+
+echo " copied into clipboard"
+echo "changeset created, check $OSM_SERVER/changeset/$changeset_id"
+echo $changeset_id | xsel -ib 
diff --git a/scripts/osm/osm.api.changeset.add b/scripts/osm/osm.api.changeset.add
new file mode 100644
index 0000000..3e4878e
--- /dev/null
+++ b/scripts/osm/osm.api.changeset.add
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+element=$(cat)
+header=$(echo $element | grep -E "<(node|way|relation)\s")
+ele_type=$(echo $header | sed -r 's/.*<(node|way|relation).*$/\1/')
+id=$(echo $header | sed -r 's/.* id=\"([^"]+)\".*$/\1/')
+
+echo $element | \
+sed -r "s/^( *<(node|way|relation).*version[^ ]+ )(.*)$/\1changeset=\"$1\">/" | \
+curl -X PUT -u $OSM_USER_PASSWD -i -T - $OSM_API/$ele_type/$id
diff --git a/scripts/osm/osm.api.changeset.close b/scripts/osm/osm.api.changeset.close
new file mode 100644
index 0000000..3b016c3
--- /dev/null
+++ b/scripts/osm/osm.api.changeset.close
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+curl -X PUT -u $OSM_USER_PASSWD -i $OSM_API/changeset/$1/close
diff --git a/scripts/osm/osm.api.changeset.update b/scripts/osm/osm.api.changeset.update
new file mode 100644
index 0000000..7b32741
--- /dev/null
+++ b/scripts/osm/osm.api.changeset.update
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+echo -n 'type comment: '
+read -r comment
+
+echo "<osm><changeset><tag k=\"comment\" v=\"$comment\"/></changeset></osm>" | \
+curl -X PUT -u $OSM_USER_PASSWD -i -T - $OSM_API/changeset/$1
diff --git a/scripts/osm/osm.api.changeset.upload b/scripts/osm/osm.api.changeset.upload
new file mode 100644
index 0000000..918a755
--- /dev/null
+++ b/scripts/osm/osm.api.changeset.upload
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+cat $2 |\
+sed -r "/<(node|way|relation)/ s/>/ changeset=\"$1\">/" |\
+curl -X POST -u $OSM_USER_PASSWD -i -T - $OSM_API/changeset/$1/upload
diff --git a/scripts/osm/osm.api.fetch b/scripts/osm/osm.api.fetch
new file mode 100644
index 0000000..8460c5f
--- /dev/null
+++ b/scripts/osm/osm.api.fetch
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+# get .osm format data
+curl -X GET $OSM_API/$1/$2 |\
+tee /tmp/osm &&\
+echo content of $1 $2 is copied into /tmp/osm > /dev/tty
diff --git a/scripts/osm/osm.api.fetch.full b/scripts/osm/osm.api.fetch.full
new file mode 100644
index 0000000..5ee78b8
--- /dev/null
+++ b/scripts/osm/osm.api.fetch.full
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+curl -X GET $OSM_API/$1/$2/full |\
+tee /tmp/osm &&\
+echo "\n" content of $1 $2 and its members are copied into /tmp/osm > /dev/tty
diff --git a/scripts/osm/osm.api.fetch.history b/scripts/osm/osm.api.fetch.history
new file mode 100644
index 0000000..bc837bd
--- /dev/null
+++ b/scripts/osm/osm.api.fetch.history
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+curl -X GET $OSM_API/$1/$2/history |\
+tee /tmp/osm &&\
+echo &&\
+echo "\n" history of $1 $2 are copied into /tmp/osm > /dev/tty
diff --git a/scripts/osm/osm.api.member.relation b/scripts/osm/osm.api.member.relation
new file mode 100644
index 0000000..5addef7
--- /dev/null
+++ b/scripts/osm/osm.api.member.relation
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+osm.api.referenced.relation $1 $2 |\
+osm.member.relation $1 $2 $3
diff --git a/scripts/osm/osm.api.referenced.relation b/scripts/osm/osm.api.referenced.relation
new file mode 100644
index 0000000..2ce0631
--- /dev/null
+++ b/scripts/osm/osm.api.referenced.relation
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+curl -X GET $OSM_API/$1/$2/relations |\
+tee /tmp/osm &&\
+echo &&\
+echo "\n" relations contain $1 $2 are copied into /tmp/osm > /dev/tty
diff --git a/scripts/osm/osm.api.referenced.way b/scripts/osm/osm.api.referenced.way
new file mode 100644
index 0000000..b62ac58
--- /dev/null
+++ b/scripts/osm/osm.api.referenced.way
@@ -0,0 +1,3 @@
+    curl -X GET $OSM_API/node/$1/ways |\
+    tee /tmp/osm &&\
+    echo ways contain node $1 are copied into /tmp/osm > /dev/tty
diff --git a/scripts/osm/osm.api.upload.to b/scripts/osm/osm.api.upload.to
new file mode 100644
index 0000000..3979220
--- /dev/null
+++ b/scripts/osm/osm.api.upload.to
@@ -0,0 +1,12 @@
+#! /bin/sh
+
+# allows multiple elements in osm body
+tee /tmp/osm |\
+osm.list.ids |\
+sed 's#.*#osm.extract \0 < /tmp/osm#g' |\
+sed "s/.*/\0 \| osm.api.changeset.add $1/g" |\
+while read -r command
+do
+    echo $command
+    source<(echo "($command &)")
+done
diff --git a/scripts/osm/osm.file.get b/scripts/osm/osm.file.get
new file mode 100644
index 0000000..e2b67b4
--- /dev/null
+++ b/scripts/osm/osm.file.get
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+file=$1; shift
+osmium getid $file $@ --output-format=osm
diff --git a/scripts/osm/osm.file.get.full b/scripts/osm/osm.file.get.full
new file mode 100644
index 0000000..50ee2f1
--- /dev/null
+++ b/scripts/osm/osm.file.get.full
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+file=$1; shift
+osmium getid $file $@ --output-format=osm --add-referenced
diff --git a/scripts/osm/osm.file.query b/scripts/osm/osm.file.query
new file mode 100644
index 0000000..4a01b5b
--- /dev/null
+++ b/scripts/osm/osm.file.query
@@ -0,0 +1,4 @@
+#! /bin/sh
+
+file=$1; shift
+osmium tags-filter $file $@ --output-format=osm --omit-referenced
diff --git a/scripts/osm/osm.get.id b/scripts/osm/osm.get.id
new file mode 100644
index 0000000..93593b5
--- /dev/null
+++ b/scripts/osm/osm.get.id
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+# $1 as [node|way|relation], $2 as id
+echo "<osm version=\"0.6\">"
+sed -nr "/^ *<$1 id=\"$2\".*/,/^ *<\/$1>/p"
+echo "</osm>"
diff --git a/scripts/osm/osm.goto b/scripts/osm/osm.goto
new file mode 100644
index 0000000..7da3394
--- /dev/null
+++ b/scripts/osm/osm.goto
@@ -0,0 +1 @@
+    xdg-open https://www.openstreetmap.org/$1/$2
diff --git a/scripts/osm/osm.help b/scripts/osm/osm.help
new file mode 100644
index 0000000..aa3b36b
--- /dev/null
+++ b/scripts/osm/osm.help
@@ -0,0 +1,34 @@
+#! /bin/sh
+
+    echo '
+COMMANDS:
+  osm.help
+  util.osm.edit
+
+  osm.api.changeset.create     create a new changeset and return its ID
+  osm.api.changeset.update     [changeset ID] update changeset with new comment
+  osm.api.changeset.upload     [changeset ID] [osc file] upload osc file to the given changeset
+  osm.api.changeset.add        [changeset ID] STDIN=osm element, add it into changeset
+  osm.api.changeset.close      [changeset ID] close the given changeset
+  osm.api.fetch                [element type] [element ID]
+  osm.api.fetch.full           like .fetch, but return with all referenced elements
+  osm.api.fetch.history        [element type] [element ID]
+  osm.api.member.relation      [element type] [element ID] [member type] get first relations reference it as member type
+  osm.api.referenced.relation  [element type] [element ID] get all relations reference this element
+  osm.api.referenced.way       [element type] [element ID] get all ways reference this element
+  osm.api.upload.to
+
+  osm.file.cat
+  osm.file.get
+  osm.file.get.full
+  osm.file.query
+  osm.list.ids
+  osm.list.tag
+  osm.list.tags
+
+  osm.get.id
+  osm.query
+  osm.member.relation
+  osm.pbf.update
+  osm.osm.update
+    '
diff --git a/scripts/osm/osm.list.ids b/scripts/osm/osm.list.ids
new file mode 100644
index 0000000..8e5d870
--- /dev/null
+++ b/scripts/osm/osm.list.ids
@@ -0,0 +1,3 @@
+#! /bin/sh
+
+sed -nr 's/.*<(node|way|relation) id=\"([^"]+)\".*/\1 \2/p'
diff --git a/scripts/osm/osm.list.tag b/scripts/osm/osm.list.tag
new file mode 100644
index 0000000..3d5157d
--- /dev/null
+++ b/scripts/osm/osm.list.tag
@@ -0,0 +1,8 @@
+    ele_pattern="(node|way|relation)"
+    sed -nr "/<$ele_pattern/,/<\/$ele_pattern/ {
+        /<tag k=\"$1\"/ {
+            s/.*v=\"([^\"]+)\".*/\1/
+            h
+        }
+        /<\/$ele_pattern/ {x;p;s/.*//;x}
+    }"
diff --git a/scripts/osm/osm.list.tags b/scripts/osm/osm.list.tags
new file mode 100644
index 0000000..6fe4da4
--- /dev/null
+++ b/scripts/osm/osm.list.tags
@@ -0,0 +1,13 @@
+    content=$(cat)
+    echo $content | osm.list.ids | tr ' ' ',' > /tmp/osm
+
+    for tag in $@
+    do
+        echo $content |\
+        osm.list.tag $tag |\
+        paste -d',' /tmp/osm - > /tmp/osm.new &&\
+        mv /tmp/osm.new /tmp/osm
+    done
+
+    cat /tmp/osm
+    echo "\ntag list is also copied into /tmp/osm" > /dev/tty
diff --git a/scripts/osm/osm.member.relation b/scripts/osm/osm.member.relation
new file mode 100644
index 0000000..f8d4598
--- /dev/null
+++ b/scripts/osm/osm.member.relation
@@ -0,0 +1,8 @@
+#! /bin/sh
+
+sed -nr "/<relation/,/<\/relation/ {
+    /<relation/ {s/.* id=\"([^\"]+)\".*/\1/;h}
+    /<member type=\"$1\" ref=\"$2\" role=\"$3\"/ {g;p;q}
+}
+$ {s/.*//;P}
+" | head -1
diff --git a/scripts/osm/osm.osc.create b/scripts/osm/osm.osc.create
new file mode 100644
index 0000000..61a4dfb
--- /dev/null
+++ b/scripts/osm/osm.osc.create
@@ -0,0 +1,17 @@
+    while read -r line
+    do
+        array=( $(echo $line) )
+
+        cat $1 |\
+        sed -nr "
+            /<$array[1] id=$array[2]/,/<\/$array[1]/ {
+            /<$array[1] id=$array[2]/ a \ \ \ \ <tag k=\"$array[3]\" v=$array[4]\/>
+            /<tag k=$array[3]/ !p
+            /<\/$array[1]/ q
+         }" >> $1.osc
+     done
+
+    sed -ir '1 i <osmChange version="0.6" generator="bash script">
+             1 i <modify>
+             $ a </modify>
+             $ a </osmChange>' $1.osc
diff --git a/scripts/osm/osm.osm.remove b/scripts/osm/osm.osm.remove
new file mode 100644
index 0000000..fbbaf0c
--- /dev/null
+++ b/scripts/osm/osm.osm.remove
@@ -0,0 +1,7 @@
+    while read -r line
+    do
+        # put element type and element ID into array
+        array=( $(echo $line) )
+        cat $1 |\
+        sed -i "/<$array[1] id=\"$array[2]\"/,/<\/$array[1]>/ d"
+    done 
diff --git a/scripts/osm/osm.pbf.update b/scripts/osm/osm.pbf.update
new file mode 100644
index 0000000..694ae4d
--- /dev/null
+++ b/scripts/osm/osm.pbf.update
@@ -0,0 +1,30 @@
+    PBF_FILE=$1
+    GEOFABRICK_SERVER=http://download.geofabrik.de/asia/taiwan-updates
+
+    # get next sequence number and store it into NEW_SEQ
+    osmium fileinfo $PBF_FILE | \
+    grep osmosis_replication_sequence_number | \
+    cut -d'=' -f2 | \
+    sed 's/$/+1/' | bc | \
+    read NEW_SEQ
+
+    # while server has osc file with given sequence number,
+    # get it and do file update
+    while
+        SEQ_PATH=$(echo $NEW_SEQ | sed -r 's/(.{1})(.{3})/00\1\/\2/')
+        STATE_URL=$GEOFABRICK_SERVER/000/$SEQ_PATH.state.txt
+        echo $STATE_URL
+        [ $(curl.code $STATE_URL) != "404" ]
+    do
+        mkdir -p changes
+        CHANGE_URL=$GEOFABRICK_SERVER/000/$SEQ_PATH.osc.gz
+        echo $CHANGE_URL
+        curl -o changes/$NEW_SEQ.osc.gz $CHANGE_URL && \
+        osmium apply-changes $PBF_FILE changes/$NEW_SEQ.osc.gz \
+            --output-header=osmosis_replication_sequence_number=$NEW_SEQ \
+            --overwrite \
+            --output $NEW_SEQ.osm.pbf
+
+        PBF_FILE=$NEW_SEQ.osm.pbf
+        ((NEW_SEQ++))
+    done
diff --git a/scripts/osm/osm.query b/scripts/osm/osm.query
new file mode 100644
index 0000000..fd48fd2
--- /dev/null
+++ b/scripts/osm/osm.query
@@ -0,0 +1 @@
+    osmium tags-filter - $@ --input-format=osm --output-format=osm --omit-referenced
diff --git a/scripts/osm/osm.village.nat_ref.makefile b/scripts/osm/osm.village.nat_ref.makefile
new file mode 100644
index 0000000..3937144
--- /dev/null
+++ b/scripts/osm/osm.village.nat_ref.makefile
@@ -0,0 +1,59 @@
+data/taiwan-latest.osm.pbf:
+        mkdir -p data
+        curl -o $@ http://download.geofabrik.de/asia/taiwan-latest.osm.pbf
+
+data/village.zip:
+        mkdir -p data
+        curl -o $@ -L http://data.moi.gov.tw/MoiOD/System/DownloadFile.aspx\?DATA\=B8AF344F-B5C6-4642-AF46-1832054399CE
+
+data/VILLAGE_MOI_1081007.shp: data/village.zip
+        @if [ ! -e $@ ]; then unzip $< -d data && rm data/*xml data/*xlsx; fi
+
+village.csv: data/taiwan-latest.osm.pbf
+        ogr2ogr $@ $< \
+                -lco GEOMETRY=AS_XY \
+                -dialect sqlite \
+                -sql "SELECT osm_id, name, other_tags, ST_PointOnSurface(geometry) \
+              FROM multipolygons \
+              WHERE admin_level = '9'"
+
+village.no_nat_ref.csv: village.csv
+        grep -v nat_ref $< > $@
+
+village.with_nat_ref.csv: village.csv
+        (head -1 $<; grep nat_ref $<) |\
+        sed -r "s/\"\"\".*nat_ref\"\"=>\"\"([^\"]+).*\"\"\"/\1/g" |\
+        sed '1s/other_tags/nat_ref/'> $@
+
+matched.csv: data/VILLAGE_MOI_1081007.shp village.no_nat_ref.csv
+        ogr2ogr $@ $(word 2,$^) \
+                -oo X_POSSIBLE_NAMES=X -oo Y_POSSIBLE_NAMES=Y \
+                -dialect sqlite \
+                -sql "SELECT osm.osm_id, gov.* \
+                      FROM 'village.no_nat_ref' osm, '$<'.VILLAGE_MOI_1081007 gov \
+                      WHERE osm.name = gov.VILLNAME AND Intersects(gov.geometry, osm.geometry)"
+
+matched.by_ref.csv: data/VILLAGE_MOI_1081007.shp village.with_nat_ref.csv
+        ogr2ogr $@ $(word 2,$^) \
+                -oo X_POSSIBLE_NAMES=X -oo Y_POSSIBLE_NAMES=Y \
+                -dialect sqlite \
+                -sql "SELECT osm.osm_id, gov.* \
+                      FROM 'village.with_nat_ref' osm, '$<'.VILLAGE_MOI_1081007 gov \
+                      WHERE osm.nat_ref = gov.VILLCODE"
+
+confilct.list: matched.csv
+        cat $< | cut -d',' -f2 | sort | uniq -d | xargs -I {} grep {} $<
+
+origin.osm: matched.csv data/taiwan-latest.osm.pbf
+        cat $< |\
+        sed 1d |\
+        cut -d'"' -f2 |\
+        sed -nr 's/.*/r\0/p' |\
+        osmium getid $(word 2,$^) --id-file - --output-format=osm > $@
+
+change.list: matched.csv
+        cat $< |\
+        sed 1d |\
+        awk -F',' '{print "relation", $$1, "nat_ref", $$2}' > $@
+
+# sed -i -r 's/([0-9]+ +)(.+)$/\1"name:en" "\2"/' diff.eng