Updated to work with tags

scripts/start-firehose.sh

@@ -1,21 +1,91 @@
 #!/bin/bash
 
-cat /config/domains-federated|grep -v "#"|while read -r host
+echo > /config/urls.txt
+echo > /config/hosts
+
+# Get federated hosts and begin to stream them
+cat /config/domains-federated | grep -v "##" | while read -r line
 do
-   if [[ "$host" != "" ]]
-   then
-      /scripts/get-stream.sh $host "federated" &
+   #filter out empty lines
+   if [[ "$line" != "" ]]; then
+      echo "Opening federated line $line"
+
+      #Check for hashtags
+      if [[ "$line" == *" #"* ]]; then
+
+         echo "$line has hashtags!"
+
+         # Get just the first field of the line, which is the host
+         host=`echo $line | cut -d " " -f 1`
+         tags=`echo $line | cut -d " " -f 1 --complement|tr "#" "\n "`
+         for tag in $tags
+         do
+            if [[ $tag != "" ]]; then
+            echo "Found tag $tag"
+            # Create a url to fetch for each tag
+            echo "https://$host/api/v1/streaming/hashtag?tag=$tag $host" >> /config/urls.txt
+            fi
+         done
+      elif [[ "$line" != *" #"* ]]; then
+         echo "$line didn't have hashtags"
+         host=$line
+         echo "https://$line/api/v1/streaming/public $line" >> /config/urls.txt
+      fi 
+      echo $host >> /config/hosts
    fi
 done
 
-cat /config/domains-local|grep -v "#"|while read -r host
+
+# Get local hosts and begin to stream them
+cat /config/domains-local | grep -v "##" | while read -r line
 do
-   if [[ "$host" != "" ]]
-   then
-      /scripts/get-stream.sh $host "local" &
+   #filter out empty lines
+   if [[ "$line" != "" ]]; then
+      echo "Opening federated line $line"
+
+      #Check for hashtags
+      if [[ "$line" == *" #"* ]]; then
+
+         echo "$line has hashtags!"
+
+         # Get just the first field of the line, which is the host
+         host=`echo $line | cut -d " " -f 1`
+         tags=`echo $line | cut -d " " -f 1 --complement|tr "#" "\n "`
+         for tag in $tags
+         do
+            if [[ $tag != "" ]]; then
+            echo "Found tag $tag"
+            # Create a url to fetch for each tag
+            echo "https://$host/api/v1/streaming/hashtag/local?tag=$tag $host" >> /config/urls.txt
+            fi
+         done
+      elif [[ "$line" != *" #"* ]]; then
+         echo "$line didn't have hashtags"
+         host=$line
+         echo "https://$line/api/v1/streaming/local $line" >> /config/urls.txt
+      fi
+      echo $host >> /config/hosts
    fi
 done
 
+cat /config/hashtags | grep -v "##" | while read -r hashtag; do
+   hashtag=`echo $hashtag | cut -d "#" -f 2`
+   sort /config/hosts | uniq -u |while read -r host; do
+      if [[ $hashtag != "" && "$host" != "" ]]; then
+         echo "https://$host/api/v1/streaming/hashtag?tag=$hashtag $host" >> /config/hashtag-urls.txt
+      fi
+   done
+done
+
+cat /config/hashtag-urls.txt >> /config/urls.txt
+
+cat /config/urls.txt | while read -r url
+do
+   echo "Opening $url to stream"
+   sleep 1s
+   ./stream-url.sh $url &
+done
+
 if [[ $runFirehose == true ]]
 then
    /scripts/run-firehose.sh &

scripts/stream-url.sh

@@ -0,0 +1,37 @@
+url=$1 #A proper URL is all that should be sent to this script
+host=$2
+
+if [[ "$url" == "" ]]
+then
+    echo "Empty url, skipping" # Exit if an empty URL was sent
+    exit 2
+fi
+
+while true # Loop endlessly
+do
+    today=`date +"%Y%m%d"`
+
+    echo "Starting to stream $url in 5 seconds"
+
+    sleep 5s;
+
+    curl -X "GET" "$url" \
+         --no-progress-meter | \
+        tee -a "/data/$today.json" | \
+        grep url | \
+        sed 's/data://g' | \
+
+     while read -r line
+     do
+
+         if [[ $line == *"uri"* ]]
+         then
+            url=`echo $line | jq .url| sed 's/\"//g'` 
+            uri=`echo $line | jq .uri| sed 's/\"//g'`
+
+            echo "STREAMING from $host $url"
+            echo $uri >> "/data/$today.uris.txt"
+
+        fi
+    done
+done