blob: a8c84b3a1b6f76db4032bfbbda4403b22dbf47e5 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
#! /bin/bash
LIST=${LIST:?}
curlOpts='-H "User-Agent: topo.tw" --insecure'
ln -sf ${LIST} $(basename $LIST)
ln -sf $0 $(basename $0)
grep '^[^#]' "${LIST}" | while read url interval tags; do
feed=${url//\//%2F}
unset next_fetch; declare -i next_fetch=0
# Get time of next fetch
if [ -f $feed ]; then
interval=$(<<<"$interval" sed 's/m/minute/; s/h/hour/; s/d/day/;')
next_fetch=$(date -d "$(date -r $feed) + $interval" +'%s')
fi
# Do not fetch feed if interval covers current time
[ $(date +'%s') -lt $next_fetch ] && continue
# Set limit of HTTP connections
while [ $(jobs -p | wc -l) -ge 10 ]; do
sleep 0.5;
done
curl -s $curlOpts $url -o $feed || echo Fail to fetch $url >&2 &
done
|
