Add archive mode where JSON is saved only

.gitignore

@@ -12,4 +12,4 @@ data/20221219.json
 config/hashtag-urls.txt
 config/urls.txt
 data/20221219.uris.txt
-./data/*
+data/*

docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.9 "
 
 services:
-  fake-firehose:
+  firehose-archive:
     build: .
     image: fakefirehose:latest
     volumes:

scripts/start-firehose.sh

@@ -82,7 +82,7 @@ cat /config/hashtag-urls.txt >> /config/urls.txt
 cat /config/urls.txt | while read -r url
 do
    echo "Opening $url to stream"
-   sleep 1s
+   sleep 0.1s
    ./stream-url.sh $url &
 done
 

scripts/stream-url.sh

@@ -12,26 +12,30 @@ do
     today=`date +"%Y%m%d"`
 
     echo "Starting to stream $url in 5 seconds"
+    echo "Archive status is $archive"
 
     sleep 5s;
 
-    curl -X "GET" "$url" \
+    if [[ $archive != "true" ]]
-         --no-progress-meter | \
+    then
-        tee -a "/data/$today.json" | \
+        curl -X "GET" "$url" \
-        grep url | \
+            --no-progress-meter | \
-        sed 's/data://g' | \
+            tee -a "/data/$today.json" | \
+            grep url | \
+            sed 's/data://g' | \
 
-     while read -r line
+        while read -r line
-     do
+        do
+            if [[ $line == *"uri"* ]]
+            then
+                url=`echo $line | jq .url| sed 's/\"//g'` 
+                uri=`echo $line | jq .uri| sed 's/\"//g'`
 
-         if [[ $line == *"uri"* ]]
+                echo "STREAMING from $host $url"
-         then
+                echo $uri >> "/data/$today.uris.txt"
-            url=`echo $line | jq .url| sed 's/\"//g'` 
+            fi
-            uri=`echo $line | jq .uri| sed 's/\"//g'`
+        done
-
+    else
-            echo "STREAMING from $host $url"
+        curl -X "GET" "$url" --no-progress-meter >> "/data/$today.$host.json"
-            echo $uri >> "/data/$today.uris.txt"
+    fi
-
-        fi
-    done
 done