Make sure that we only process the same url once

2023-01-14 13:51:31 +01:00 · 2023-01-14 13:51:31 +01:00 · 237ed1303d
parent e66b06503d
commit 237ed1303d
1 changed files with 1 additions and 1 deletions
--- a/scripts/start-firehose.sh
+++ b/scripts/start-firehose.sh
@ -80,7 +80,7 @@ done

 cat /config/hashtag-urls.txt >> /config/urls.txt

-cat /config/urls.txt | while read -r url
+sort -u /config/urls.txt | while read -r url
 do
   echo "[INFO] Opening $url to stream"
   sleep $streamDelay