Merge pull request #5144 from pixelfed/staging

Update AP fetch service and domain service

app/Services/ActivityPubFetchService.php

@@ -2,76 +2,132 @@
 
 namespace App\Services;
 
-use Illuminate\Support\Facades\Http;
-use App\Profile;
-use App\Util\ActivityPub\Helpers;
 use App\Util\ActivityPub\HttpSignature;
+use Cache;
 use Illuminate\Http\Client\ConnectionException;
 use Illuminate\Http\Client\RequestException;
+use Illuminate\Support\Facades\Http;
 
 class ActivityPubFetchService
 {
+    const CACHE_KEY = 'pf:services:apfetchs:';
+
     public static function get($url, $validateUrl = true)
     {
-        if($validateUrl === true) {
+        if (! self::validateUrl($url)) {
-            if(!Helpers::validateUrl($url)) {
+            return false;
-                return 0;
-            }
         }
+        $domain = parse_url($url, PHP_URL_HOST);
+        if (! $domain) {
+            return false;
+        }
+        $domainKey = base64_encode($domain);
+        $urlKey = hash('sha256', $url);
+        $key = self::CACHE_KEY.$domainKey.':'.$urlKey;
 
-        $baseHeaders = [
+        return Cache::remember($key, 3600, function () use ($url) {
-            'Accept' => 'application/activity+json, application/ld+json',
+            $baseHeaders = [
-        ];
+                'Accept' => 'application/activity+json',
+            ];
 
-        $headers = HttpSignature::instanceActorSign($url, false, $baseHeaders, 'get');
+            $headers = HttpSignature::instanceActorSign($url, false, $baseHeaders, 'get');
-        $headers['Accept'] = 'application/activity+json, application/ld+json';
+            $headers['Accept'] = 'application/activity+json';
-        $headers['User-Agent'] = 'PixelFedBot/1.0.0 (Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')';
+            $headers['User-Agent'] = 'PixelFedBot/1.0.0 (Pixelfed/'.config('pixelfed.version').'; +'.config('app.url').')';
 
-        try {
+            try {
-            $res = Http::withoutVerifying()
+                $res = Http::withOptions([
-                ->withOptions([
                     'allow_redirects' => [
                         'max' => 2,
                         'protocols' => ['https'],
-                    ]
+                    ]])
-                ])
+                    ->withHeaders($headers)
-                ->withHeaders($headers)
+                    ->timeout(30)
-                ->timeout(30)
+                    ->connectTimeout(5)
-                ->connectTimeout(5)
+                    ->retry(3, 500)
-                ->retry(3, 500)
+                    ->get($url);
-                ->get($url);
+            } catch (RequestException $e) {
-        } catch (RequestException $e) {
+                return;
-            return;
+            } catch (ConnectionException $e) {
-        } catch (ConnectionException $e) {
+                return;
-            return;
+            } catch (Exception $e) {
-        } catch (Exception $e) {
+                return;
-            return;
+            }
+
+            if (! $res->ok()) {
+                return;
+            }
+
+            if (! $res->hasHeader('Content-Type')) {
+                return;
+            }
+
+            $acceptedTypes = [
+                'application/activity+json; charset=utf-8',
+                'application/activity+json',
+                'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+            ];
+
+            $contentType = $res->getHeader('Content-Type')[0];
+
+            if (! $contentType) {
+                return;
+            }
+
+            if (! in_array($contentType, $acceptedTypes)) {
+                return;
+            }
+
+            return $res->body();
+        });
+    }
+
+    public static function validateUrl($url)
+    {
+        if (is_array($url)) {
+            $url = $url[0];
         }
 
-        if(!$res->ok()) {
+        $localhosts = [
-            return;
+            '127.0.0.1', 'localhost', '::1',
-        }
-
-        if(!$res->hasHeader('Content-Type')) {
-            return;
-        }
-
-        $acceptedTypes = [
-            'application/activity+json; charset=utf-8',
-            'application/activity+json',
-            'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'
         ];
 
-        $contentType = $res->getHeader('Content-Type')[0];
+        if (strtolower(mb_substr($url, 0, 8)) !== 'https://') {
-
+            return false;
-        if(!$contentType) {
-            return;
         }
 
-        if(!in_array($contentType, $acceptedTypes)) {
+        if (substr_count($url, '://') !== 1) {
-            return;
+            return false;
         }
 
-        return $res->body();
+        if (mb_substr($url, 0, 8) !== 'https://') {
+            $url = 'https://'.substr($url, 8);
+        }
+
+        $valid = filter_var($url, FILTER_VALIDATE_URL);
+
+        if (! $valid) {
+            return false;
+        }
+
+        $host = parse_url($valid, PHP_URL_HOST);
+
+        if (in_array($host, $localhosts)) {
+            return false;
+        }
+
+        if (config('security.url.verify_dns')) {
+            if (DomainService::hasValidDns($host) === false) {
+                return false;
+            }
+        }
+
+        if (app()->environment() === 'production') {
+            $bannedInstances = InstanceService::getBannedDomains();
+            if (in_array($host, $bannedInstances)) {
+                return false;
+            }
+        }
+
+        return $url;
     }
 }

app/Services/DomainService.php

@@ -3,25 +3,30 @@
 namespace App\Services;
 
 use Illuminate\Support\Facades\Cache;
-use Illuminate\Support\Facades\Redis;
 
 class DomainService
 {
-	const CACHE_KEY = 'pf:services:domains:';
+    const CACHE_KEY = 'pf:services:domains:';
 
     public static function hasValidDns($domain)
     {
-        if(!$domain || !strlen($domain) || strpos($domain, '.') == -1) {
+        if (! $domain || ! strlen($domain) || strpos($domain, '.') == -1) {
             return false;
         }
 
-        if(config('security.url.trusted_domains')) {
+        if (config('security.url.trusted_domains')) {
-            if(in_array($domain, explode(',', config('security.url.trusted_domains')))) {
+            if (in_array($domain, explode(',', config('security.url.trusted_domains')))) {
                 return true;
             }
         }
 
-        return Cache::remember(self::CACHE_KEY . 'valid-dns:' . $domain, 14400, function() use($domain) {
+        $valid = filter_var($domain, FILTER_VALIDATE_DOMAIN, FILTER_FLAG_HOSTNAME);
+
+        if (! $valid) {
+            return false;
+        }
+
+        return Cache::remember(self::CACHE_KEY.'valid-dns:'.$domain, 1800, function () use ($domain) {
             return count(dns_get_record($domain, DNS_A | DNS_AAAA)) > 0;
         });
     }