chris/fake-firehose
1
0
Fork 0
forked from mirror/fake-firehose
Code Pull requests Activity

Added archive mode and some basic health checks

Browse source
This commit is contained in:
Raynor 2022-12-22 14:06:43 +00:00
parent 543f896d52
commit 9d3f7f4873
1 changed files with 37 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
scripts/stream-url.sh
View file

@ -1,23 +1,41 @@
url=$1 #A proper URL is all that should be sent to this script url=$1 #A proper URL is all that should be sent to this script
host=$2 host=$2
errors=0
if [[ "$url" == "" ]] if [[ "$url" == "" ]]
then then
echo "Empty url, skipping" # Exit if an empty URL was sent echo "[WARN] Empty url, skipping" # Exit if an empty URL was sent
exit 2 exit 2
fi fi
# if [[ "$checkUrl" != *"200"* ]]
# then
# echo "[WARN] Server threw an error, skipping"
# fi
# Check to see if domain name resolves. If not, exist
if [[ ! `dig $host +short` ]]
then
echo "[WARN] DNS Lookup failed for $host, skipping"
fi
echo "[INFO] Archive is $archive"
while true # Loop endlessly while true # Loop endlessly
do do
today=`date +"%Y%m%d"` today=`date +"%Y%m%d"`
echo "Starting to stream $url in 5 seconds" echo "[INFO] Starting to stream $url in 5 seconds"
echo "Archive status is $archive" echo "[INFO] Archive status is $archive"
sleep 5s; sleep 5s;
# Im archive mode we'll only fetch the json stream to save resources from jq and sed
if [[ $archive != "true" ]] if [[ $archive != "true" ]]
then then
#Not in archive mode
curl -X "GET" "$url" \ curl -X "GET" "$url" \
--no-progress-meter | \ --no-progress-meter | \
tee -a "/data/$today.json" | \ tee -a "/data/$today.json" | \
@ -31,10 +49,11 @@ do
url=`echo $line | jq .url| sed 's/\"//g'` url=`echo $line | jq .url| sed 's/\"//g'`
uri=`echo $line | jq .uri| sed 's/\"//g'` uri=`echo $line | jq .uri| sed 's/\"//g'`
echo "STREAMING from $host $url" echo "[INFO] STREAMING from $host $url"
echo $uri >> "/data/$today.uris.txt" echo $uri >> "/data/$today.uris.txt"
fi fi
done done
# In archive mode
else else
if [[ ! -d "/data/$today/" ]] if [[ ! -d "/data/$today/" ]]
@ -45,12 +64,23 @@ do
curl -X "GET" "$url" --no-progress-meter >> "/data/$today/$today.$host.json" curl -X "GET" "$url" --no-progress-meter >> "/data/$today/$today.$host.json"
fi fi
# If we are in archive mode only, then back off if there is a curl error. # Basic exponential backoff
if [[ $archive == "true" ]] ((++errors))
sleepseconds=$((errors*errors))
# Don't allow a back off for more than 5 minutes.
# Because we expect this container to reset occasionally to kill hanging curl processes
# a graceful exit will wait for all scripts to stop. So, it will take at least as long as $sleepseconds
# to stop.
if [[ $sleepseconds -gt 299 ]]
then then
sleep 5m; sleepseconds=300
fi fi
sleep $sleepseconds;
echo "[WARN] Streaming abrubtly stopped for $host, streaming will pause for $sleepseconds seconds before retrying."
done done
## Exit 0 by default ## Exit 0 by default