blob: 8dff837c9bef913b6819e63e95f899e7f5afaf8c (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
|
#! /bin/bash
cd /srv/rss
mkdir -p feeds
LIST=${LIST:?}
curlOpts='--insecure'
cat "${LIST}" | while read url interval tags; do
feed=feeds/${url//\//%2F}
unset next_fetch; declare -i next_fetch=0
# Get time of next fetch
if [ -f $feed ]; then
interval=$(<<<"$interval" sed 's/m/minute/; s/h/hour/; s/d/day/;')
next_fetch=$(date -d "$(date -r $feed) + $interval" +'%s')
fi
# Do not fetch feed if interval covers current time
[ $(date +'%s') -lt $next_fetch ] && continue
# Set limit of HTTP connections
while [ $(jobs -p | wc -l) -ge 10 ]; do
sleep 0.5;
done
curl -s $curlOpts $url -o $feed || echo Fail to fetch $url >&2 &
done
|
