Add pagnation to fetch even if user's gists >100

1 files changed, 48 insertions, 27 deletions

diff --git a/gist b/gist
index fe3ee42..b9cb192 100755
--- a/gist
+++ b/gist
@@ -58,6 +58,7 @@ configuredClient=""
 NAME=${GISTSCRIPT:-$(basename $0)} #show hint and helper message with current script name
 GITHUB_API=https://api.github.com
 CONFIG=~/.config/gist.conf; mkdir -p ~/.config
+per_page=100
 
 INDEX_FORMAT=('index' 'url' 'tags_string' 'blob_code' 'file_array' 'file_num' 'comment_num' 'author' 'created_at' 'updated_at' 'description')
 TAG_CHAR='-_[:alnum:]'
@@ -125,7 +126,7 @@ http_method() {
   case "$configuredClient" in
     curl)   [[ -n $token ]] && header_opt="--header" header="Authorization: token $token"
       [[ $METHOD =~ (POST|PATCH) ]] && data_opt='--data'
-      curl -X "$METHOD" -A curl -s $header_opt "$header" $data_opt "@$http_data" "$@" ;;
+      curl -X "$METHOD" -A curl -s $header_opt "$header" $data_opt ${HEADER:+-D $HEADER} "@$http_data" "$@" ;;
     wget)   [[ -n $token ]] && header_opt="--header" header="Authorization: token $token"
       [[ $METHOD =~ (POST|PATCH) ]] && data_opt='--body-file'
       wget --method="$METHOD" -qO- $header_opt "$header" $data_opt "$http_data" "$@" ;;
@@ -356,29 +357,30 @@ _grep_content() {
 
 # Parse JSON object of the result of gist fetch
 _parse_gists() {
-    _process_json '
+  _process_json '
 raw = json.load(sys.stdin)
 for gist in raw:
-    print(gist["html_url"], end=" ")
-    print(gist["public"], end=" ")
-    print(",".join(file["raw_url"] for file in gist["files"].values()), end=" ")
-    print(",".join(file["filename"].replace(" ", "-") + "@" + str(file["language"]).replace(" ", "-") for file in gist["files"].values()), end=" ")
-    print(len(gist["files"]), end=" ")
-    print(gist["comments"], end=" ")
-    print(gist["owner"]["login"], end=" ")
-    print(gist["created_at"], end=" ")
-    print(gist["updated_at"], end=" ")
-    print(gist["description"])
-    '
+  print(gist["html_url"], end=" ")
+  print(gist["public"], end=" ")
+  print(",".join(file["raw_url"] for file in gist["files"].values()), end=" ")
+  print(",".join(file["filename"].replace(" ", "-") + "@" + str(file["language"]).replace(" ", "-") for file in gist["files"].values()), end=" ")
+  print(len(gist["files"]), end=" ")
+  print(gist["comments"], end=" ")
+  print(gist["owner"]["login"], end=" ")
+  print(gist["created_at"], end=" ")
+  print(gist["updated_at"], end=" ")
+  print(gist["description"])
+  '
 }
 
 # Parse response from 'gist fetch' to the format for index file
 _parse_response() {
+  local num=$start_from_num
   _parse_gists \
   | tac | nl -s' ' \
   | while read -r "${INDEX_FORMAT[@]:0:2}" public file_url_array "${INDEX_FORMAT[@]:4:7}"; do
     local private_prefix=''; [[ $public == 'False' ]] && private_prefix=p
-    [[ -n $1 ]] && local index=${1}; index=${private_prefix}${prefix}${index}
+    [[ -n $num ]] && local index=${num} && num=$(($num+1)); index=${private_prefix}${prefix}${index}
 
     local blob_code=$(echo "$file_url_array" | tr ',' '\n' | sed -E -e 's#.*raw/(.*)/.*#\1#' | sort | cut -c -7 | paste -s -d '-' -)
     file_array=${file_array//@None/@Text}
@@ -393,27 +395,45 @@ _parse_response() {
 }
 
 # Get latest list of gists from Github API
-# TODO pagnation for more than 100 gists
 _fetch_gists() {
-  echo "fetching $user's gists from $GITHUB_API..."
-  echo
   local route="users/$user/gists"
   local prifix=''
   if [[ $mark == s ]]; then
     route='gists/starred'
     prefix=s
-    extra="s0 https://gist.github.com/b0d2e7e67aa50298fdf8111ae7466b56 #bash,#gist NONE README.md@Markdown,gist@Shell 2 30 typebrook 2019-12-26T06:49:40Z 2020-05-15T13:00:31Z [bash-gist] A bash script for gist management"
   fi
 
-  result=$(http_method GET $GITHUB_API/$route?per_page=100 | prefix=$prefix _parse_response)
-  result=$(printf "${extra:+$extra\n}$result")
-  [[ -z $result ]] && echo 'Not a single valid gist' && return 0
+  # set global variable HEADER in http_method, so prevent using pipe
+  HEADER=$(tmp_file HEADER) 
+  HEADER=$HEADER http_method GET $GITHUB_API/$route${1} | start_from_num=$2 prefix=$prefix _parse_response
+}
+
+# consider if HEADER is not exist
+_fetch_gists_with_pagnation() {
+  echo "fetching $user's gists from $GITHUB_API..."
+  echo
+
+  local fetched_records=$(tmp_file fetched)
+  _fetch_gists "?per_page=$per_page" >> $fetched_records
+
+  while true; do
+    local next_page=''
+    [[ -e $HEADER ]] && next_page=$(sed -Ene '/^[lL]ink: / s/.+page=([[:digit:]]+)>; rel=\"next\".+/\1/p' $HEADER)
+    [[ -z $next_page ]] && break
+    printf "%-4s gists fetched\n" $(( ($next_page -1) * $per_page ))
+
+    _fetch_gists "?per_page=$per_page&page=$next_page" $(( $(wc -l <$fetched_records) +1 )) >> $fetched_records 
+  done || return 1
+
+  [[ ! -s $fetched_records ]] && echo 'Not a single valid gist' && return 0
 
-  sed -i'' -Ee "/^$mark/ d" $INDEX && echo "$result" >> $INDEX
+  sed -i'' -Ee "/^$mark/ d" $INDEX; cat $fetched_records >> $INDEX
   hint=$hint _show_list
 
   [[ $auto_sync != false ]] && (_sync_repos &> /dev/null &)
   true
+  #extra="s0 https://gist.github.com/b0d2e7e67aa50298fdf8111ae7466b56 #bash,#gist NONE README.md@Markdown,gist@Shell 2 30 typebrook 2019-12-26T06:49:40Z 2020-05-15T13:00:31Z [bash-gist] A bash script for gist management"
+  #result=$(printf "${extra:+$extra\n}$result")
 }
 
 # Fetch gists for a given user
@@ -601,7 +621,8 @@ _push_to_remote() {
     git add . && git commit -m 'update'
   fi
   if [[ -n $(git cherry) ]]; then
-    git push origin master && (hint=false _fetch_gists &> /dev/null &)
+    # FIXME only fetch one gist
+    git push origin master && (hint=false _fetch_gists_with_pagnation &> /dev/null &)
   fi
 }
 
@@ -669,7 +690,7 @@ _create_gist() {
   | _gist_body > "$http_data" \
   && http_method POST $GITHUB_API/gists \
   | xargs -I{} -0 echo "[{}]" \
-  | _parse_response $(( $(sed -e '/^s/ d' $INDEX | wc -l) +1 )) \
+  | start_from_num=$(( $(sed -e '/^s/ d' $INDEX | wc -l) +1 )) _parse_response \
   | tee -a $INDEX \
   | cut -d' ' -f2 | sed -E -e 's#.*/##' \
   | (xargs -I{} git clone "$(_repo_url {})" $folder/{} &> /dev/null &)
@@ -701,7 +722,7 @@ _edit_gist() {
   echo '{' \"description\": \""${DESC//\"/\\\"}"\" '}' > "$http_data"
   new_record=$(http_method PATCH "$GITHUB_API/gists/$GIST_ID" \
                | sed -e '1 s/^/[/; $ s/$/]/' \
-               | _parse_response "${index#[[:alpha:]]}" )
+               | start_from_index="${index#[[:alpha:]]}" _parse_response)
   [[ -n $new_record ]] && sed -i'' -E -e "/^$index / s^.+^$new_record^" $INDEX \
   && hint=false mark="$index " _show_list \
   || echo 'Fail to modify gist description'
@@ -811,7 +832,7 @@ _access_last_index() {
 }
 
 _apply_config "$@" || exit 1
-if [[ $init ]]; then _fetch_gists; exit 0; fi
+if [[ $init ]]; then _fetch_gists_with_pagnation; exit 0; fi
 case "$1" in
   "")
     _show_list ;;
@@ -821,7 +842,7 @@ case "$1" in
     mark=.; _show_list ;;
   fetch | f)
     [[ $2 =~ ^(s|star)$ ]] && mark=s || mark=[^s]
-    _fetch_gists ;;
+    _fetch_gists_with_pagnation ;;
   new | n)
     shift
     _create_gist "$@" ;;