From 48657936432fbbea797c4822234f41f739fd05e9 Mon Sep 17 00:00:00 2001 From: typebrook Date: Sun, 17 May 2020 00:57:15 +0800 Subject: Add pagnation to fetch even if user's gists >100 --- gist | 75 +++++++++++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 27 deletions(-) diff --git a/gist b/gist index fe3ee42..b9cb192 100755 --- a/gist +++ b/gist @@ -58,6 +58,7 @@ configuredClient="" NAME=${GISTSCRIPT:-$(basename $0)} #show hint and helper message with current script name GITHUB_API=https://api.github.com CONFIG=~/.config/gist.conf; mkdir -p ~/.config +per_page=100 INDEX_FORMAT=('index' 'url' 'tags_string' 'blob_code' 'file_array' 'file_num' 'comment_num' 'author' 'created_at' 'updated_at' 'description') TAG_CHAR='-_[:alnum:]' @@ -125,7 +126,7 @@ http_method() { case "$configuredClient" in curl) [[ -n $token ]] && header_opt="--header" header="Authorization: token $token" [[ $METHOD =~ (POST|PATCH) ]] && data_opt='--data' - curl -X "$METHOD" -A curl -s $header_opt "$header" $data_opt "@$http_data" "$@" ;; + curl -X "$METHOD" -A curl -s $header_opt "$header" $data_opt ${HEADER:+-D $HEADER} "@$http_data" "$@" ;; wget) [[ -n $token ]] && header_opt="--header" header="Authorization: token $token" [[ $METHOD =~ (POST|PATCH) ]] && data_opt='--body-file' wget --method="$METHOD" -qO- $header_opt "$header" $data_opt "$http_data" "$@" ;; @@ -356,29 +357,30 @@ _grep_content() { # Parse JSON object of the result of gist fetch _parse_gists() { - _process_json ' + _process_json ' raw = json.load(sys.stdin) for gist in raw: - print(gist["html_url"], end=" ") - print(gist["public"], end=" ") - print(",".join(file["raw_url"] for file in gist["files"].values()), end=" ") - print(",".join(file["filename"].replace(" ", "-") + "@" + str(file["language"]).replace(" ", "-") for file in gist["files"].values()), end=" ") - print(len(gist["files"]), end=" ") - print(gist["comments"], end=" ") - print(gist["owner"]["login"], end=" ") - print(gist["created_at"], end=" ") - print(gist["updated_at"], end=" ") - print(gist["description"]) - ' + print(gist["html_url"], end=" ") + print(gist["public"], end=" ") + print(",".join(file["raw_url"] for file in gist["files"].values()), end=" ") + print(",".join(file["filename"].replace(" ", "-") + "@" + str(file["language"]).replace(" ", "-") for file in gist["files"].values()), end=" ") + print(len(gist["files"]), end=" ") + print(gist["comments"], end=" ") + print(gist["owner"]["login"], end=" ") + print(gist["created_at"], end=" ") + print(gist["updated_at"], end=" ") + print(gist["description"]) + ' } # Parse response from 'gist fetch' to the format for index file _parse_response() { + local num=$start_from_num _parse_gists \ | tac | nl -s' ' \ | while read -r "${INDEX_FORMAT[@]:0:2}" public file_url_array "${INDEX_FORMAT[@]:4:7}"; do local private_prefix=''; [[ $public == 'False' ]] && private_prefix=p - [[ -n $1 ]] && local index=${1}; index=${private_prefix}${prefix}${index} + [[ -n $num ]] && local index=${num} && num=$(($num+1)); index=${private_prefix}${prefix}${index} local blob_code=$(echo "$file_url_array" | tr ',' '\n' | sed -E -e 's#.*raw/(.*)/.*#\1#' | sort | cut -c -7 | paste -s -d '-' -) file_array=${file_array//@None/@Text} @@ -393,27 +395,45 @@ _parse_response() { } # Get latest list of gists from Github API -# TODO pagnation for more than 100 gists _fetch_gists() { - echo "fetching $user's gists from $GITHUB_API..." - echo local route="users/$user/gists" local prifix='' if [[ $mark == s ]]; then route='gists/starred' prefix=s - extra="s0 https://gist.github.com/b0d2e7e67aa50298fdf8111ae7466b56 #bash,#gist NONE README.md@Markdown,gist@Shell 2 30 typebrook 2019-12-26T06:49:40Z 2020-05-15T13:00:31Z [bash-gist] A bash script for gist management" fi - result=$(http_method GET $GITHUB_API/$route?per_page=100 | prefix=$prefix _parse_response) - result=$(printf "${extra:+$extra\n}$result") - [[ -z $result ]] && echo 'Not a single valid gist' && return 0 + # set global variable HEADER in http_method, so prevent using pipe + HEADER=$(tmp_file HEADER) + HEADER=$HEADER http_method GET $GITHUB_API/$route${1} | start_from_num=$2 prefix=$prefix _parse_response +} + +# consider if HEADER is not exist +_fetch_gists_with_pagnation() { + echo "fetching $user's gists from $GITHUB_API..." + echo + + local fetched_records=$(tmp_file fetched) + _fetch_gists "?per_page=$per_page" >> $fetched_records + + while true; do + local next_page='' + [[ -e $HEADER ]] && next_page=$(sed -Ene '/^[lL]ink: / s/.+page=([[:digit:]]+)>; rel=\"next\".+/\1/p' $HEADER) + [[ -z $next_page ]] && break + printf "%-4s gists fetched\n" $(( ($next_page -1) * $per_page )) + + _fetch_gists "?per_page=$per_page&page=$next_page" $(( $(wc -l <$fetched_records) +1 )) >> $fetched_records + done || return 1 + + [[ ! -s $fetched_records ]] && echo 'Not a single valid gist' && return 0 - sed -i'' -Ee "/^$mark/ d" $INDEX && echo "$result" >> $INDEX + sed -i'' -Ee "/^$mark/ d" $INDEX; cat $fetched_records >> $INDEX hint=$hint _show_list [[ $auto_sync != false ]] && (_sync_repos &> /dev/null &) true + #extra="s0 https://gist.github.com/b0d2e7e67aa50298fdf8111ae7466b56 #bash,#gist NONE README.md@Markdown,gist@Shell 2 30 typebrook 2019-12-26T06:49:40Z 2020-05-15T13:00:31Z [bash-gist] A bash script for gist management" + #result=$(printf "${extra:+$extra\n}$result") } # Fetch gists for a given user @@ -601,7 +621,8 @@ _push_to_remote() { git add . && git commit -m 'update' fi if [[ -n $(git cherry) ]]; then - git push origin master && (hint=false _fetch_gists &> /dev/null &) + # FIXME only fetch one gist + git push origin master && (hint=false _fetch_gists_with_pagnation &> /dev/null &) fi } @@ -669,7 +690,7 @@ _create_gist() { | _gist_body > "$http_data" \ && http_method POST $GITHUB_API/gists \ | xargs -I{} -0 echo "[{}]" \ - | _parse_response $(( $(sed -e '/^s/ d' $INDEX | wc -l) +1 )) \ + | start_from_num=$(( $(sed -e '/^s/ d' $INDEX | wc -l) +1 )) _parse_response \ | tee -a $INDEX \ | cut -d' ' -f2 | sed -E -e 's#.*/##' \ | (xargs -I{} git clone "$(_repo_url {})" $folder/{} &> /dev/null &) @@ -701,7 +722,7 @@ _edit_gist() { echo '{' \"description\": \""${DESC//\"/\\\"}"\" '}' > "$http_data" new_record=$(http_method PATCH "$GITHUB_API/gists/$GIST_ID" \ | sed -e '1 s/^/[/; $ s/$/]/' \ - | _parse_response "${index#[[:alpha:]]}" ) + | start_from_index="${index#[[:alpha:]]}" _parse_response) [[ -n $new_record ]] && sed -i'' -E -e "/^$index / s^.+^$new_record^" $INDEX \ && hint=false mark="$index " _show_list \ || echo 'Fail to modify gist description' @@ -811,7 +832,7 @@ _access_last_index() { } _apply_config "$@" || exit 1 -if [[ $init ]]; then _fetch_gists; exit 0; fi +if [[ $init ]]; then _fetch_gists_with_pagnation; exit 0; fi case "$1" in "") _show_list ;; @@ -821,7 +842,7 @@ case "$1" in mark=.; _show_list ;; fetch | f) [[ $2 =~ ^(s|star)$ ]] && mark=s || mark=[^s] - _fetch_gists ;; + _fetch_gists_with_pagnation ;; new | n) shift _create_gist "$@" ;; -- cgit v1.2.3-70-g09d2