From 9233cd8f5b27f36c528dd97c35068081b8d9ae47 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 11 Jul 2023 02:27:08 -0600 Subject: [PATCH 01/10] Add migration --- ...40_add_show_reblogs_to_followers_table.php | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 database/migrations/2023_07_11_080040_add_show_reblogs_to_followers_table.php diff --git a/database/migrations/2023_07_11_080040_add_show_reblogs_to_followers_table.php b/database/migrations/2023_07_11_080040_add_show_reblogs_to_followers_table.php new file mode 100644 index 000000000..d84433747 --- /dev/null +++ b/database/migrations/2023_07_11_080040_add_show_reblogs_to_followers_table.php @@ -0,0 +1,30 @@ +boolean('show_reblogs')->default(true)->index()->after('local_following'); + $table->boolean('notify')->default(false)->index()->after('show_reblogs'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('followers', function (Blueprint $table) { + $table->dropColumn('show_reblogs'); + $table->dropColumn('notify'); + }); + } +}; From 1cc6274ac0bdafdfbdc8183cc55212298ad9a0d1 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 13 Jul 2023 21:38:18 -0600 Subject: [PATCH 02/10] Update rate limits, fixes #4537 --- app/Http/Controllers/Api/ApiV1Controller.php | 6 +++--- app/Http/Controllers/Api/ApiV2Controller.php | 2 +- app/Http/Controllers/ComposeController.php | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index 3e67632a7..5d3207d2a 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -1621,7 +1621,7 @@ class ApiV1Controller extends Controller $limitReached = Cache::remember($limitKey, $limitTtl, function() use($user) { $dailyLimit = Media::whereUserId($user->id)->where('created_at', '>', now()->subDays(1))->count(); - return $dailyLimit >= 250; + return $dailyLimit >= 1250; }); abort_if($limitReached == true, 429); @@ -1826,7 +1826,7 @@ class ApiV1Controller extends Controller $limitReached = Cache::remember($limitKey, $limitTtl, function() use($user) { $dailyLimit = Media::whereUserId($user->id)->where('created_at', '>', now()->subDays(1))->count(); - return $dailyLimit >= 250; + return $dailyLimit >= 1250; }); abort_if($limitReached == true, 429); @@ -2838,7 +2838,7 @@ class ApiV1Controller extends Controller ->where('created_at', '>', now()->subDays(1)) ->count(); - return $dailyLimit >= 100; + return $dailyLimit >= 1000; }); abort_if($limitReached == true, 429); diff --git a/app/Http/Controllers/Api/ApiV2Controller.php b/app/Http/Controllers/Api/ApiV2Controller.php index 63c63c56f..757e14dce 100644 --- a/app/Http/Controllers/Api/ApiV2Controller.php +++ b/app/Http/Controllers/Api/ApiV2Controller.php @@ -225,7 +225,7 @@ class ApiV2Controller extends Controller $limitReached = Cache::remember($limitKey, $limitTtl, function() use($user) { $dailyLimit = Media::whereUserId($user->id)->where('created_at', '>', now()->subDays(1))->count(); - return $dailyLimit >= 250; + return $dailyLimit >= 1250; }); abort_if($limitReached == true, 429); diff --git a/app/Http/Controllers/ComposeController.php b/app/Http/Controllers/ComposeController.php index 7a3614f3d..54526ffe8 100644 --- a/app/Http/Controllers/ComposeController.php +++ b/app/Http/Controllers/ComposeController.php @@ -98,7 +98,7 @@ class ComposeController extends Controller $limitReached = Cache::remember($limitKey, $limitTtl, function() use($user) { $dailyLimit = Media::whereUserId($user->id)->where('created_at', '>', now()->subDays(1))->count(); - return $dailyLimit >= 250; + return $dailyLimit >= 1250; }); abort_if($limitReached == true, 429); @@ -190,7 +190,7 @@ class ComposeController extends Controller $limitReached = Cache::remember($limitKey, $limitTtl, function() use($user) { $dailyLimit = Media::whereUserId($user->id)->where('created_at', '>', now()->subDays(1))->count(); - return $dailyLimit >= 500; + return $dailyLimit >= 1500; }); abort_if($limitReached == true, 429); @@ -499,7 +499,7 @@ class ComposeController extends Controller ->where('created_at', '>', now()->subDays(1)) ->count(); - return $dailyLimit >= 100; + return $dailyLimit >= 1000; }); abort_if($limitReached == true, 429); From 4b2c66f5578911ad94b749cda1a801701bf168b9 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 13 Jul 2023 22:06:21 -0600 Subject: [PATCH 03/10] Update Services, use zpopmin on predis --- app/Services/LikeService.php | 4 +--- app/Services/NetworkTimelineService.php | 4 +--- app/Services/PublicTimelineService.php | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/app/Services/LikeService.php b/app/Services/LikeService.php index 34a2417d0..f0ea1ac57 100644 --- a/app/Services/LikeService.php +++ b/app/Services/LikeService.php @@ -24,9 +24,7 @@ class LikeService { public static function setAdd($profileId, $statusId) { if(self::setCount($profileId) > 400) { - if(config('database.redis.client') === 'phpredis') { - Redis::zpopmin(self::CACHE_SET_KEY . $profileId); - } + Redis::zpopmin(self::CACHE_SET_KEY . $profileId); } return Redis::zadd(self::CACHE_SET_KEY . $profileId, $statusId, $statusId); diff --git a/app/Services/NetworkTimelineService.php b/app/Services/NetworkTimelineService.php index 570899017..9aea47af4 100644 --- a/app/Services/NetworkTimelineService.php +++ b/app/Services/NetworkTimelineService.php @@ -49,9 +49,7 @@ class NetworkTimelineService public static function add($val) { if(self::count() > config('instance.timeline.network.cache_dropoff')) { - if(config('database.redis.client') === 'phpredis') { - Redis::zpopmin(self::CACHE_KEY); - } + Redis::zpopmin(self::CACHE_KEY); } return Redis::zadd(self::CACHE_KEY, $val, $val); diff --git a/app/Services/PublicTimelineService.php b/app/Services/PublicTimelineService.php index e1275065c..f2658e4b1 100644 --- a/app/Services/PublicTimelineService.php +++ b/app/Services/PublicTimelineService.php @@ -49,9 +49,7 @@ class PublicTimelineService { public static function add($val) { if(self::count() > 400) { - if(config('database.redis.client') === 'phpredis') { - Redis::zpopmin(self::CACHE_KEY); - } + Redis::zpopmin(self::CACHE_KEY); } return Redis::zadd(self::CACHE_KEY, $val, $val); From 9fa6b3f7aa1ae03bc94ce1d57606511df6ca7689 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Thu, 13 Jul 2023 23:11:19 -0600 Subject: [PATCH 04/10] Update Inbox, allow storing Create->Note activities without any local followers, disabled by default --- app/Util/ActivityPub/Inbox.php | 3 +- config/federation.php | 87 ++++++++++++++++++---------------- 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/app/Util/ActivityPub/Inbox.php b/app/Util/ActivityPub/Inbox.php index 0caf8f25d..fe9beeb27 100644 --- a/app/Util/ActivityPub/Inbox.php +++ b/app/Util/ActivityPub/Inbox.php @@ -281,7 +281,8 @@ class Inbox } if($actor->followers_count == 0) { - if(FollowerService::followerCount($actor->id, true) == 0) { + if(config('federation.activitypub.ingest.store_notes_without_followers')) { + } else if(FollowerService::followerCount($actor->id, true) == 0) { return; } } diff --git a/config/federation.php b/config/federation.php index 4b6795687..773d3d16b 100644 --- a/config/federation.php +++ b/config/federation.php @@ -2,56 +2,59 @@ return [ - /* - |-------------------------------------------------------------------------- - | ActivityPub - |-------------------------------------------------------------------------- - | - | ActivityPub configuration - | - */ - 'activitypub' => [ - 'enabled' => env('ACTIVITY_PUB', false), - 'outbox' => env('AP_OUTBOX', true), - 'inbox' => env('AP_INBOX', true), - 'sharedInbox' => env('AP_SHAREDINBOX', true), + /* + |-------------------------------------------------------------------------- + | ActivityPub + |-------------------------------------------------------------------------- + | + | ActivityPub configuration + | + */ + 'activitypub' => [ + 'enabled' => env('ACTIVITY_PUB', false), + 'outbox' => env('AP_OUTBOX', true), + 'inbox' => env('AP_INBOX', true), + 'sharedInbox' => env('AP_SHAREDINBOX', true), - 'remoteFollow' => env('AP_REMOTE_FOLLOW', true), + 'remoteFollow' => env('AP_REMOTE_FOLLOW', true), - 'delivery' => [ - 'timeout' => env('ACTIVITYPUB_DELIVERY_TIMEOUT', 30.0), - 'concurrency' => env('ACTIVITYPUB_DELIVERY_CONCURRENCY', 10), - 'logger' => [ - 'enabled' => env('AP_LOGGER_ENABLED', false), - 'driver' => 'log' - ] - ] - ], + 'delivery' => [ + 'timeout' => env('ACTIVITYPUB_DELIVERY_TIMEOUT', 30.0), + 'concurrency' => env('ACTIVITYPUB_DELIVERY_CONCURRENCY', 10), + 'logger' => [ + 'enabled' => env('AP_LOGGER_ENABLED', false), + 'driver' => 'log' + ] + ], - 'atom' => [ - 'enabled' => env('ATOM_FEEDS', true), - ], + 'ingest' => [ + 'store_notes_without_followers' => env('AP_INGEST_STORE_NOTES_WITHOUT_FOLLOWERS', false), + ], + ], - 'avatars' => [ - 'store_local' => env('REMOTE_AVATARS', true), - ], + 'atom' => [ + 'enabled' => env('ATOM_FEEDS', true), + ], - 'nodeinfo' => [ - 'enabled' => env('NODEINFO', true), - ], + 'avatars' => [ + 'store_local' => env('REMOTE_AVATARS', true), + ], - 'webfinger' => [ - 'enabled' => env('WEBFINGER', true) - ], + 'nodeinfo' => [ + 'enabled' => env('NODEINFO', true), + ], - 'network_timeline' => env('PF_NETWORK_TIMELINE', true), - 'network_timeline_days_falloff' => env('PF_NETWORK_TIMELINE_DAYS_FALLOFF', 2), + 'webfinger' => [ + 'enabled' => env('WEBFINGER', true) + ], - 'custom_emoji' => [ - 'enabled' => env('CUSTOM_EMOJI', false), + 'network_timeline' => env('PF_NETWORK_TIMELINE', true), + 'network_timeline_days_falloff' => env('PF_NETWORK_TIMELINE_DAYS_FALLOFF', 2), - // max size in bytes, default is 2mb - 'max_size' => env('CUSTOM_EMOJI_MAX_SIZE', 2000000), - ] + 'custom_emoji' => [ + 'enabled' => env('CUSTOM_EMOJI', false), + // max size in bytes, default is 2mb + 'max_size' => env('CUSTOM_EMOJI_MAX_SIZE', 2000000), + ], ]; From 0704c7e05ee4caa4835dc3c9a5538f0b17e133b1 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 14 Jul 2023 01:20:26 -0600 Subject: [PATCH 05/10] Update AP Helpers, preserve admin unlisted state before adding to NetworkTimelineService --- app/Util/ActivityPub/Helpers.php | 1451 +++++++++++++++--------------- 1 file changed, 730 insertions(+), 721 deletions(-) diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index a61e1d572..243b92482 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -4,14 +4,14 @@ namespace App\Util\ActivityPub; use DB, Cache, Purify, Storage, Request, Validator; use App\{ - Activity, - Follower, - Instance, - Like, - Media, - Notification, - Profile, - Status + Activity, + Follower, + Instance, + Like, + Media, + Notification, + Profile, + Status }; use Zttp\Zttp; use Carbon\Carbon; @@ -44,762 +44,771 @@ use App\Services\UserFilterService; class Helpers { - public static function validateObject($data) - { - $verbs = ['Create', 'Announce', 'Like', 'Follow', 'Delete', 'Accept', 'Reject', 'Undo', 'Tombstone']; - - $valid = Validator::make($data, [ - 'type' => [ - 'required', - 'string', - Rule::in($verbs) - ], - 'id' => 'required|string', - 'actor' => 'required|string|url', - 'object' => 'required', - 'object.type' => 'required_if:type,Create', - 'object.attributedTo' => 'required_if:type,Create|url', - 'published' => 'required_if:type,Create|date' - ])->passes(); - - return $valid; - } - - public static function verifyAttachments($data) - { - if(!isset($data['object']) || empty($data['object'])) { - $data = ['object'=>$data]; - } - - $activity = $data['object']; - - $mimeTypes = explode(',', config_cache('pixelfed.media_types')); - $mediaTypes = in_array('video/mp4', $mimeTypes) ? ['Document', 'Image', 'Video'] : ['Document', 'Image']; - - // Peertube - // $mediaTypes = in_array('video/mp4', $mimeTypes) ? ['Document', 'Image', 'Video', 'Link'] : ['Document', 'Image']; - - if(!isset($activity['attachment']) || empty($activity['attachment'])) { - return false; - } - - // peertube - // $attachment = is_array($activity['url']) ? - // collect($activity['url']) - // ->filter(function($media) { - // return $media['type'] == 'Link' && $media['mediaType'] == 'video/mp4'; - // }) - // ->take(1) - // ->values() - // ->toArray()[0] : $activity['attachment']; - - $attachment = $activity['attachment']; - - $valid = Validator::make($attachment, [ - '*.type' => [ - 'required', - 'string', - Rule::in($mediaTypes) - ], - '*.url' => 'required|url', - '*.mediaType' => [ - 'required', - 'string', - Rule::in($mimeTypes) - ], - '*.name' => 'sometimes|nullable|string' - ])->passes(); - - return $valid; - } - - public static function normalizeAudience($data, $localOnly = true) - { - if(!isset($data['to'])) { - return; - } - - $audience = []; - $audience['to'] = []; - $audience['cc'] = []; - $scope = 'private'; - - if(is_array($data['to']) && !empty($data['to'])) { - foreach ($data['to'] as $to) { - if($to == 'https://www.w3.org/ns/activitystreams#Public') { - $scope = 'public'; - continue; - } - $url = $localOnly ? self::validateLocalUrl($to) : self::validateUrl($to); - if($url != false) { - array_push($audience['to'], $url); - } - } - } - - if(is_array($data['cc']) && !empty($data['cc'])) { - foreach ($data['cc'] as $cc) { - if($cc == 'https://www.w3.org/ns/activitystreams#Public') { - $scope = 'unlisted'; - continue; - } - $url = $localOnly ? self::validateLocalUrl($cc) : self::validateUrl($cc); - if($url != false) { - array_push($audience['cc'], $url); - } - } - } - $audience['scope'] = $scope; - return $audience; - } - - public static function userInAudience($profile, $data) - { - $audience = self::normalizeAudience($data); - $url = $profile->permalink(); - return in_array($url, $audience['to']) || in_array($url, $audience['cc']); - } - - public static function validateUrl($url) - { - if(is_array($url)) { - $url = $url[0]; - } - - $hash = hash('sha256', $url); - $key = "helpers:url:valid:sha256-{$hash}"; - $ttl = now()->addMinutes(5); - - $valid = Cache::remember($key, $ttl, function() use($url) { - $localhosts = [ - '127.0.0.1', 'localhost', '::1' - ]; - - if(mb_substr($url, 0, 8) !== 'https://') { - return false; - } - - $valid = filter_var($url, FILTER_VALIDATE_URL); - - if(!$valid) { - return false; - } - - $host = parse_url($valid, PHP_URL_HOST); - - // if(count(dns_get_record($host, DNS_A | DNS_AAAA)) == 0) { - // return false; - // } - - if(config('costar.enabled') == true) { - if( - (config('costar.domain.block') != null && Str::contains($host, config('costar.domain.block')) == true) || - (config('costar.actor.block') != null && in_array($url, config('costar.actor.block')) == true) - ) { - return false; - } - } - - if(app()->environment() === 'production') { - $bannedInstances = InstanceService::getBannedDomains(); - if(in_array($host, $bannedInstances)) { - return false; - } - } - - - if(in_array($host, $localhosts)) { - return false; - } - - return $url; - }); - - return $valid; - } - - public static function validateLocalUrl($url) - { - $url = self::validateUrl($url); - if($url == true) { - $domain = config('pixelfed.domain.app'); - $host = parse_url($url, PHP_URL_HOST); - $url = $domain === $host ? $url : false; - return $url; - } - return false; - } - - public static function zttpUserAgent() - { - $version = config('pixelfed.version'); - $url = config('app.url'); - return [ - 'Accept' => 'application/activity+json', - 'User-Agent' => "(Pixelfed/{$version}; +{$url})", - ]; - } - - public static function fetchFromUrl($url = false) - { - if(self::validateUrl($url) == false) { - return; - } - - $hash = hash('sha256', $url); - $key = "helpers:url:fetcher:sha256-{$hash}"; - $ttl = now()->addMinutes(15); - - return Cache::remember($key, $ttl, function() use($url) { - $res = ActivityPubFetchService::get($url); - if(!$res || empty($res)) { - return false; - } - $res = json_decode($res, true, 8); - if(json_last_error() == JSON_ERROR_NONE) { - return $res; - } else { - return false; - } - }); - } - - public static function fetchProfileFromUrl($url) - { - return self::fetchFromUrl($url); - } - - public static function pluckval($val) - { - if(is_string($val)) { - return $val; - } - - if(is_array($val)) { - return !empty($val) ? $val[0] : null; - } - - return null; - } - - public static function statusFirstOrFetch($url, $replyTo = false) - { - $url = self::validateUrl($url); - if($url == false) { - return; - } - - $host = parse_url($url, PHP_URL_HOST); - $local = config('pixelfed.domain.app') == $host ? true : false; - - if($local) { - $id = (int) last(explode('/', $url)); - return Status::whereNotIn('scope', ['draft','archived'])->findOrFail($id); - } - - $cached = Status::whereNotIn('scope', ['draft','archived']) - ->whereUri($url) - ->orWhere('object_url', $url) - ->first(); - - if($cached) { - return $cached; - } - - $res = self::fetchFromUrl($url); - - if(!$res || empty($res) || isset($res['error']) || !isset($res['@context']) || !isset($res['published']) ) { - return; - } - - if(isset($res['object'])) { - $activity = $res; - } else { - $activity = ['object' => $res]; - } - - $scope = 'private'; - - $cw = isset($res['sensitive']) ? (bool) $res['sensitive'] : false; - - if(isset($res['to']) == true) { - if(is_array($res['to']) && in_array('https://www.w3.org/ns/activitystreams#Public', $res['to'])) { - $scope = 'public'; - } - if(is_string($res['to']) && 'https://www.w3.org/ns/activitystreams#Public' == $res['to']) { - $scope = 'public'; - } - } - - if(isset($res['cc']) == true) { - if(is_array($res['cc']) && in_array('https://www.w3.org/ns/activitystreams#Public', $res['cc'])) { - $scope = 'unlisted'; - } - if(is_string($res['cc']) && 'https://www.w3.org/ns/activitystreams#Public' == $res['cc']) { - $scope = 'unlisted'; - } - } - - if(config('costar.enabled') == true) { - $blockedKeywords = config('costar.keyword.block'); - if($blockedKeywords !== null) { - $keywords = config('costar.keyword.block'); - foreach($keywords as $kw) { - if(Str::contains($res['content'], $kw) == true) { - return; - } - } - } - - $unlisted = config('costar.domain.unlisted'); - if(in_array(parse_url($url, PHP_URL_HOST), $unlisted) == true) { - $unlisted = true; - $scope = 'unlisted'; - } else { - $unlisted = false; - } - - $cwDomains = config('costar.domain.cw'); - if(in_array(parse_url($url, PHP_URL_HOST), $cwDomains) == true) { - $cw = true; - } - } - - $id = isset($res['id']) ? self::pluckval($res['id']) : self::pluckval($url); - $idDomain = parse_url($id, PHP_URL_HOST); - $urlDomain = parse_url($url, PHP_URL_HOST); - - if(!self::validateUrl($id)) { - return; - } - - if(!isset($activity['object']['attributedTo'])) { - return; - } - - $attributedTo = is_string($activity['object']['attributedTo']) ? - $activity['object']['attributedTo'] : - (is_array($activity['object']['attributedTo']) ? - collect($activity['object']['attributedTo']) - ->filter(function($o) { - return $o && isset($o['type']) && $o['type'] == 'Person'; - }) - ->pluck('id') - ->first() : null - ); - - if($attributedTo) { - $actorDomain = parse_url($attributedTo, PHP_URL_HOST); - if(!self::validateUrl($attributedTo) || - $idDomain !== $actorDomain || - $actorDomain !== $urlDomain - ) - { - return; - } - } - - if($idDomain !== $urlDomain) { - return; - } - - $profile = self::profileFirstOrNew($attributedTo); - - if(!$profile) { - return; - } - - if(isset($activity['object']['inReplyTo']) && !empty($activity['object']['inReplyTo']) || $replyTo == true) { - $reply_to = self::statusFirstOrFetch(self::pluckval($activity['object']['inReplyTo']), false); - if($reply_to) { - $blocks = UserFilterService::blocks($reply_to->profile_id); - if(in_array($profile->id, $blocks)) { - return; - } - } - $reply_to = optional($reply_to)->id; - } else { - $reply_to = null; - } - $ts = self::pluckval($res['published']); - - if($scope == 'public' && in_array($urlDomain, InstanceService::getUnlistedDomains())) { - $scope = 'unlisted'; - } - - if(in_array($urlDomain, InstanceService::getNsfwDomains())) { - $cw = true; - } - - if($res['type'] === 'Question') { - $status = self::storePoll( - $profile, - $res, - $url, - $ts, - $reply_to, - $cw, - $scope, - $id - ); - return $status; - } else { - $status = self::storeStatus($url, $profile, $res); - } - - return $status; - } - - public static function storeStatus($url, $profile, $activity) - { - $id = isset($activity['id']) ? self::pluckval($activity['id']) : self::pluckval($activity['url']); - $url = isset($activity['url']) && is_string($activity['url']) ? self::pluckval($activity['url']) : self::pluckval($id); - $idDomain = parse_url($id, PHP_URL_HOST); - $urlDomain = parse_url($url, PHP_URL_HOST); - if(!self::validateUrl($id) || !self::validateUrl($url)) { - return; - } - - $reply_to = self::getReplyTo($activity); - - $ts = self::pluckval($activity['published']); - $scope = self::getScope($activity, $url); - $cw = self::getSensitive($activity, $url); - $pid = is_object($profile) ? $profile->id : (is_array($profile) ? $profile['id'] : null); - $commentsDisabled = isset($activity['commentsEnabled']) ? !boolval($activity['commentsEnabled']) : false; - - if(!$pid) { - return; - } - - $status = Status::updateOrCreate( - [ - 'uri' => $url - ], [ - 'profile_id' => $pid, - 'url' => $url, - 'object_url' => $id, - 'caption' => isset($activity['content']) ? Purify::clean(strip_tags($activity['content'])) : null, - 'rendered' => isset($activity['content']) ? Purify::clean($activity['content']) : null, - 'created_at' => Carbon::parse($ts)->tz('UTC'), - 'in_reply_to_id' => $reply_to, - 'local' => false, - 'is_nsfw' => $cw, - 'scope' => $scope, - 'visibility' => $scope, - 'cw_summary' => ($cw == true && isset($activity['summary']) ? - Purify::clean(strip_tags($activity['summary'])) : null), - 'comments_disabled' => $commentsDisabled - ] - ); - - if($reply_to == null) { - self::importNoteAttachment($activity, $status); - } else { - if(isset($activity['attachment']) && !empty($activity['attachment'])) { - self::importNoteAttachment($activity, $status); - } - StatusReplyPipeline::dispatch($status); - } - - if(isset($activity['tag']) && is_array($activity['tag']) && !empty($activity['tag'])) { - StatusTagsPipeline::dispatch($activity, $status); - } - - if( config('instance.timeline.network.cached') && - $status->in_reply_to_id === null && - $status->reblog_of_id === null && - in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) && - $status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old'))) && - (config('instance.hide_nsfw_on_public_feeds') == true ? $status->is_nsfw == false : true) - ) { - $filteredDomains = collect(InstanceService::getBannedDomains()) + public static function validateObject($data) + { + $verbs = ['Create', 'Announce', 'Like', 'Follow', 'Delete', 'Accept', 'Reject', 'Undo', 'Tombstone']; + + $valid = Validator::make($data, [ + 'type' => [ + 'required', + 'string', + Rule::in($verbs) + ], + 'id' => 'required|string', + 'actor' => 'required|string|url', + 'object' => 'required', + 'object.type' => 'required_if:type,Create', + 'object.attributedTo' => 'required_if:type,Create|url', + 'published' => 'required_if:type,Create|date' + ])->passes(); + + return $valid; + } + + public static function verifyAttachments($data) + { + if(!isset($data['object']) || empty($data['object'])) { + $data = ['object'=>$data]; + } + + $activity = $data['object']; + + $mimeTypes = explode(',', config_cache('pixelfed.media_types')); + $mediaTypes = in_array('video/mp4', $mimeTypes) ? ['Document', 'Image', 'Video'] : ['Document', 'Image']; + + // Peertube + // $mediaTypes = in_array('video/mp4', $mimeTypes) ? ['Document', 'Image', 'Video', 'Link'] : ['Document', 'Image']; + + if(!isset($activity['attachment']) || empty($activity['attachment'])) { + return false; + } + + // peertube + // $attachment = is_array($activity['url']) ? + // collect($activity['url']) + // ->filter(function($media) { + // return $media['type'] == 'Link' && $media['mediaType'] == 'video/mp4'; + // }) + // ->take(1) + // ->values() + // ->toArray()[0] : $activity['attachment']; + + $attachment = $activity['attachment']; + + $valid = Validator::make($attachment, [ + '*.type' => [ + 'required', + 'string', + Rule::in($mediaTypes) + ], + '*.url' => 'required|url', + '*.mediaType' => [ + 'required', + 'string', + Rule::in($mimeTypes) + ], + '*.name' => 'sometimes|nullable|string' + ])->passes(); + + return $valid; + } + + public static function normalizeAudience($data, $localOnly = true) + { + if(!isset($data['to'])) { + return; + } + + $audience = []; + $audience['to'] = []; + $audience['cc'] = []; + $scope = 'private'; + + if(is_array($data['to']) && !empty($data['to'])) { + foreach ($data['to'] as $to) { + if($to == 'https://www.w3.org/ns/activitystreams#Public') { + $scope = 'public'; + continue; + } + $url = $localOnly ? self::validateLocalUrl($to) : self::validateUrl($to); + if($url != false) { + array_push($audience['to'], $url); + } + } + } + + if(is_array($data['cc']) && !empty($data['cc'])) { + foreach ($data['cc'] as $cc) { + if($cc == 'https://www.w3.org/ns/activitystreams#Public') { + $scope = 'unlisted'; + continue; + } + $url = $localOnly ? self::validateLocalUrl($cc) : self::validateUrl($cc); + if($url != false) { + array_push($audience['cc'], $url); + } + } + } + $audience['scope'] = $scope; + return $audience; + } + + public static function userInAudience($profile, $data) + { + $audience = self::normalizeAudience($data); + $url = $profile->permalink(); + return in_array($url, $audience['to']) || in_array($url, $audience['cc']); + } + + public static function validateUrl($url) + { + if(is_array($url)) { + $url = $url[0]; + } + + $hash = hash('sha256', $url); + $key = "helpers:url:valid:sha256-{$hash}"; + $ttl = now()->addMinutes(5); + + $valid = Cache::remember($key, $ttl, function() use($url) { + $localhosts = [ + '127.0.0.1', 'localhost', '::1' + ]; + + if(mb_substr($url, 0, 8) !== 'https://') { + return false; + } + + $valid = filter_var($url, FILTER_VALIDATE_URL); + + if(!$valid) { + return false; + } + + $host = parse_url($valid, PHP_URL_HOST); + + // if(count(dns_get_record($host, DNS_A | DNS_AAAA)) == 0) { + // return false; + // } + + if(config('costar.enabled') == true) { + if( + (config('costar.domain.block') != null && Str::contains($host, config('costar.domain.block')) == true) || + (config('costar.actor.block') != null && in_array($url, config('costar.actor.block')) == true) + ) { + return false; + } + } + + if(app()->environment() === 'production') { + $bannedInstances = InstanceService::getBannedDomains(); + if(in_array($host, $bannedInstances)) { + return false; + } + } + + + if(in_array($host, $localhosts)) { + return false; + } + + return $url; + }); + + return $valid; + } + + public static function validateLocalUrl($url) + { + $url = self::validateUrl($url); + if($url == true) { + $domain = config('pixelfed.domain.app'); + $host = parse_url($url, PHP_URL_HOST); + $url = $domain === $host ? $url : false; + return $url; + } + return false; + } + + public static function zttpUserAgent() + { + $version = config('pixelfed.version'); + $url = config('app.url'); + return [ + 'Accept' => 'application/activity+json', + 'User-Agent' => "(Pixelfed/{$version}; +{$url})", + ]; + } + + public static function fetchFromUrl($url = false) + { + if(self::validateUrl($url) == false) { + return; + } + + $hash = hash('sha256', $url); + $key = "helpers:url:fetcher:sha256-{$hash}"; + $ttl = now()->addMinutes(15); + + return Cache::remember($key, $ttl, function() use($url) { + $res = ActivityPubFetchService::get($url); + if(!$res || empty($res)) { + return false; + } + $res = json_decode($res, true, 8); + if(json_last_error() == JSON_ERROR_NONE) { + return $res; + } else { + return false; + } + }); + } + + public static function fetchProfileFromUrl($url) + { + return self::fetchFromUrl($url); + } + + public static function pluckval($val) + { + if(is_string($val)) { + return $val; + } + + if(is_array($val)) { + return !empty($val) ? $val[0] : null; + } + + return null; + } + + public static function statusFirstOrFetch($url, $replyTo = false) + { + $url = self::validateUrl($url); + if($url == false) { + return; + } + + $host = parse_url($url, PHP_URL_HOST); + $local = config('pixelfed.domain.app') == $host ? true : false; + + if($local) { + $id = (int) last(explode('/', $url)); + return Status::whereNotIn('scope', ['draft','archived'])->findOrFail($id); + } + + $cached = Status::whereNotIn('scope', ['draft','archived']) + ->whereUri($url) + ->orWhere('object_url', $url) + ->first(); + + if($cached) { + return $cached; + } + + $res = self::fetchFromUrl($url); + + if(!$res || empty($res) || isset($res['error']) || !isset($res['@context']) || !isset($res['published']) ) { + return; + } + + if(isset($res['object'])) { + $activity = $res; + } else { + $activity = ['object' => $res]; + } + + $scope = 'private'; + + $cw = isset($res['sensitive']) ? (bool) $res['sensitive'] : false; + + if(isset($res['to']) == true) { + if(is_array($res['to']) && in_array('https://www.w3.org/ns/activitystreams#Public', $res['to'])) { + $scope = 'public'; + } + if(is_string($res['to']) && 'https://www.w3.org/ns/activitystreams#Public' == $res['to']) { + $scope = 'public'; + } + } + + if(isset($res['cc']) == true) { + if(is_array($res['cc']) && in_array('https://www.w3.org/ns/activitystreams#Public', $res['cc'])) { + $scope = 'unlisted'; + } + if(is_string($res['cc']) && 'https://www.w3.org/ns/activitystreams#Public' == $res['cc']) { + $scope = 'unlisted'; + } + } + + if(config('costar.enabled') == true) { + $blockedKeywords = config('costar.keyword.block'); + if($blockedKeywords !== null) { + $keywords = config('costar.keyword.block'); + foreach($keywords as $kw) { + if(Str::contains($res['content'], $kw) == true) { + return; + } + } + } + + $unlisted = config('costar.domain.unlisted'); + if(in_array(parse_url($url, PHP_URL_HOST), $unlisted) == true) { + $unlisted = true; + $scope = 'unlisted'; + } else { + $unlisted = false; + } + + $cwDomains = config('costar.domain.cw'); + if(in_array(parse_url($url, PHP_URL_HOST), $cwDomains) == true) { + $cw = true; + } + } + + $id = isset($res['id']) ? self::pluckval($res['id']) : self::pluckval($url); + $idDomain = parse_url($id, PHP_URL_HOST); + $urlDomain = parse_url($url, PHP_URL_HOST); + + if(!self::validateUrl($id)) { + return; + } + + if(!isset($activity['object']['attributedTo'])) { + return; + } + + $attributedTo = is_string($activity['object']['attributedTo']) ? + $activity['object']['attributedTo'] : + (is_array($activity['object']['attributedTo']) ? + collect($activity['object']['attributedTo']) + ->filter(function($o) { + return $o && isset($o['type']) && $o['type'] == 'Person'; + }) + ->pluck('id') + ->first() : null + ); + + if($attributedTo) { + $actorDomain = parse_url($attributedTo, PHP_URL_HOST); + if(!self::validateUrl($attributedTo) || + $idDomain !== $actorDomain || + $actorDomain !== $urlDomain + ) + { + return; + } + } + + if($idDomain !== $urlDomain) { + return; + } + + $profile = self::profileFirstOrNew($attributedTo); + + if(!$profile) { + return; + } + + if(isset($activity['object']['inReplyTo']) && !empty($activity['object']['inReplyTo']) || $replyTo == true) { + $reply_to = self::statusFirstOrFetch(self::pluckval($activity['object']['inReplyTo']), false); + if($reply_to) { + $blocks = UserFilterService::blocks($reply_to->profile_id); + if(in_array($profile->id, $blocks)) { + return; + } + } + $reply_to = optional($reply_to)->id; + } else { + $reply_to = null; + } + $ts = self::pluckval($res['published']); + + if($scope == 'public' && in_array($urlDomain, InstanceService::getUnlistedDomains())) { + $scope = 'unlisted'; + } + + if(in_array($urlDomain, InstanceService::getNsfwDomains())) { + $cw = true; + } + + if($res['type'] === 'Question') { + $status = self::storePoll( + $profile, + $res, + $url, + $ts, + $reply_to, + $cw, + $scope, + $id + ); + return $status; + } else { + $status = self::storeStatus($url, $profile, $res); + } + + return $status; + } + + public static function storeStatus($url, $profile, $activity) + { + $id = isset($activity['id']) ? self::pluckval($activity['id']) : self::pluckval($activity['url']); + $url = isset($activity['url']) && is_string($activity['url']) ? self::pluckval($activity['url']) : self::pluckval($id); + $idDomain = parse_url($id, PHP_URL_HOST); + $urlDomain = parse_url($url, PHP_URL_HOST); + if(!self::validateUrl($id) || !self::validateUrl($url)) { + return; + } + + $reply_to = self::getReplyTo($activity); + + $ts = self::pluckval($activity['published']); + $scope = self::getScope($activity, $url); + $cw = self::getSensitive($activity, $url); + $pid = is_object($profile) ? $profile->id : (is_array($profile) ? $profile['id'] : null); + $isUnlisted = is_object($profile) ? $profile->unlisted : (is_array($profile) ? $profile['unlisted'] : false); + $commentsDisabled = isset($activity['commentsEnabled']) ? !boolval($activity['commentsEnabled']) : false; + + if(!$pid) { + return; + } + + if($scope == 'public') { + if($isUnlisted == true) { + $scope = 'unlisted'; + } + } + + $status = Status::updateOrCreate( + [ + 'uri' => $url + ], [ + 'profile_id' => $pid, + 'url' => $url, + 'object_url' => $id, + 'caption' => isset($activity['content']) ? Purify::clean(strip_tags($activity['content'])) : null, + 'rendered' => isset($activity['content']) ? Purify::clean($activity['content']) : null, + 'created_at' => Carbon::parse($ts)->tz('UTC'), + 'in_reply_to_id' => $reply_to, + 'local' => false, + 'is_nsfw' => $cw, + 'scope' => $scope, + 'visibility' => $scope, + 'cw_summary' => ($cw == true && isset($activity['summary']) ? + Purify::clean(strip_tags($activity['summary'])) : null), + 'comments_disabled' => $commentsDisabled + ] + ); + + if($reply_to == null) { + self::importNoteAttachment($activity, $status); + } else { + if(isset($activity['attachment']) && !empty($activity['attachment'])) { + self::importNoteAttachment($activity, $status); + } + StatusReplyPipeline::dispatch($status); + } + + if(isset($activity['tag']) && is_array($activity['tag']) && !empty($activity['tag'])) { + StatusTagsPipeline::dispatch($activity, $status); + } + + if( config('instance.timeline.network.cached') && + $status->in_reply_to_id === null && + $status->reblog_of_id === null && + in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) && + $status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old'))) && + (config('instance.hide_nsfw_on_public_feeds') == true ? $status->is_nsfw == false : true) + ) { + $filteredDomains = collect(InstanceService::getBannedDomains()) ->merge(InstanceService::getUnlistedDomains()) ->unique() ->values() ->toArray(); if(!in_array($urlDomain, $filteredDomains)) { - NetworkTimelineService::add($status->id); + if(!$isUnlisted) { + NetworkTimelineService::add($status->id); + } } - } + } - IncrementPostCount::dispatch($pid)->onQueue('low'); + IncrementPostCount::dispatch($pid)->onQueue('low'); - return $status; - } + return $status; + } - public static function getSensitive($activity, $url) - { - $id = isset($activity['id']) ? self::pluckval($activity['id']) : self::pluckval($url); - $url = isset($activity['url']) ? self::pluckval($activity['url']) : $id; - $urlDomain = parse_url($url, PHP_URL_HOST); + public static function getSensitive($activity, $url) + { + $id = isset($activity['id']) ? self::pluckval($activity['id']) : self::pluckval($url); + $url = isset($activity['url']) ? self::pluckval($activity['url']) : $id; + $urlDomain = parse_url($url, PHP_URL_HOST); - $cw = isset($activity['sensitive']) ? (bool) $activity['sensitive'] : false; + $cw = isset($activity['sensitive']) ? (bool) $activity['sensitive'] : false; - if(in_array($urlDomain, InstanceService::getNsfwDomains())) { - $cw = true; - } + if(in_array($urlDomain, InstanceService::getNsfwDomains())) { + $cw = true; + } - return $cw; - } + return $cw; + } - public static function getReplyTo($activity) - { - $reply_to = null; - $inReplyTo = isset($activity['inReplyTo']) && !empty($activity['inReplyTo']) ? - self::pluckval($activity['inReplyTo']) : - false; + public static function getReplyTo($activity) + { + $reply_to = null; + $inReplyTo = isset($activity['inReplyTo']) && !empty($activity['inReplyTo']) ? + self::pluckval($activity['inReplyTo']) : + false; - if($inReplyTo) { - $reply_to = self::statusFirstOrFetch($inReplyTo); - if($reply_to) { - $reply_to = optional($reply_to)->id; - } - } else { - $reply_to = null; - } + if($inReplyTo) { + $reply_to = self::statusFirstOrFetch($inReplyTo); + if($reply_to) { + $reply_to = optional($reply_to)->id; + } + } else { + $reply_to = null; + } - return $reply_to; - } + return $reply_to; + } - public static function getScope($activity, $url) - { - $id = isset($activity['id']) ? self::pluckval($activity['id']) : self::pluckval($url); - $url = isset($activity['url']) ? self::pluckval($activity['url']) : self::pluckval($id); - $urlDomain = parse_url(self::pluckval($url), PHP_URL_HOST); - $scope = 'private'; + public static function getScope($activity, $url) + { + $id = isset($activity['id']) ? self::pluckval($activity['id']) : self::pluckval($url); + $url = isset($activity['url']) ? self::pluckval($activity['url']) : self::pluckval($id); + $urlDomain = parse_url(self::pluckval($url), PHP_URL_HOST); + $scope = 'private'; - if(isset($activity['to']) == true) { - if(is_array($activity['to']) && in_array('https://www.w3.org/ns/activitystreams#Public', $activity['to'])) { - $scope = 'public'; - } - if(is_string($activity['to']) && 'https://www.w3.org/ns/activitystreams#Public' == $activity['to']) { - $scope = 'public'; - } - } + if(isset($activity['to']) == true) { + if(is_array($activity['to']) && in_array('https://www.w3.org/ns/activitystreams#Public', $activity['to'])) { + $scope = 'public'; + } + if(is_string($activity['to']) && 'https://www.w3.org/ns/activitystreams#Public' == $activity['to']) { + $scope = 'public'; + } + } - if(isset($activity['cc']) == true) { - if(is_array($activity['cc']) && in_array('https://www.w3.org/ns/activitystreams#Public', $activity['cc'])) { - $scope = 'unlisted'; - } - if(is_string($activity['cc']) && 'https://www.w3.org/ns/activitystreams#Public' == $activity['cc']) { - $scope = 'unlisted'; - } - } + if(isset($activity['cc']) == true) { + if(is_array($activity['cc']) && in_array('https://www.w3.org/ns/activitystreams#Public', $activity['cc'])) { + $scope = 'unlisted'; + } + if(is_string($activity['cc']) && 'https://www.w3.org/ns/activitystreams#Public' == $activity['cc']) { + $scope = 'unlisted'; + } + } - if($scope == 'public' && in_array($urlDomain, InstanceService::getUnlistedDomains())) { - $scope = 'unlisted'; - } + if($scope == 'public' && in_array($urlDomain, InstanceService::getUnlistedDomains())) { + $scope = 'unlisted'; + } - return $scope; - } + return $scope; + } - private static function storePoll($profile, $res, $url, $ts, $reply_to, $cw, $scope, $id) - { - if(!isset($res['endTime']) || !isset($res['oneOf']) || !is_array($res['oneOf']) || count($res['oneOf']) > 4) { - return; - } + private static function storePoll($profile, $res, $url, $ts, $reply_to, $cw, $scope, $id) + { + if(!isset($res['endTime']) || !isset($res['oneOf']) || !is_array($res['oneOf']) || count($res['oneOf']) > 4) { + return; + } - $options = collect($res['oneOf'])->map(function($option) { - return $option['name']; - })->toArray(); + $options = collect($res['oneOf'])->map(function($option) { + return $option['name']; + })->toArray(); - $cachedTallies = collect($res['oneOf'])->map(function($option) { - return $option['replies']['totalItems'] ?? 0; - })->toArray(); + $cachedTallies = collect($res['oneOf'])->map(function($option) { + return $option['replies']['totalItems'] ?? 0; + })->toArray(); - $status = new Status; - $status->profile_id = $profile->id; - $status->url = isset($res['url']) ? $res['url'] : $url; - $status->uri = isset($res['url']) ? $res['url'] : $url; - $status->object_url = $id; - $status->caption = strip_tags($res['content']); - $status->rendered = Purify::clean($res['content']); - $status->created_at = Carbon::parse($ts)->tz('UTC'); - $status->in_reply_to_id = null; - $status->local = false; - $status->is_nsfw = $cw; - $status->scope = 'draft'; - $status->visibility = 'draft'; - $status->cw_summary = $cw == true && isset($res['summary']) ? - Purify::clean(strip_tags($res['summary'])) : null; - $status->save(); + $status = new Status; + $status->profile_id = $profile->id; + $status->url = isset($res['url']) ? $res['url'] : $url; + $status->uri = isset($res['url']) ? $res['url'] : $url; + $status->object_url = $id; + $status->caption = strip_tags($res['content']); + $status->rendered = Purify::clean($res['content']); + $status->created_at = Carbon::parse($ts)->tz('UTC'); + $status->in_reply_to_id = null; + $status->local = false; + $status->is_nsfw = $cw; + $status->scope = 'draft'; + $status->visibility = 'draft'; + $status->cw_summary = $cw == true && isset($res['summary']) ? + Purify::clean(strip_tags($res['summary'])) : null; + $status->save(); - $poll = new Poll; - $poll->status_id = $status->id; - $poll->profile_id = $status->profile_id; - $poll->poll_options = $options; - $poll->cached_tallies = $cachedTallies; - $poll->votes_count = array_sum($cachedTallies); - $poll->expires_at = now()->parse($res['endTime']); - $poll->last_fetched_at = now(); - $poll->save(); + $poll = new Poll; + $poll->status_id = $status->id; + $poll->profile_id = $status->profile_id; + $poll->poll_options = $options; + $poll->cached_tallies = $cachedTallies; + $poll->votes_count = array_sum($cachedTallies); + $poll->expires_at = now()->parse($res['endTime']); + $poll->last_fetched_at = now(); + $poll->save(); - $status->type = 'poll'; - $status->scope = $scope; - $status->visibility = $scope; - $status->save(); + $status->type = 'poll'; + $status->scope = $scope; + $status->visibility = $scope; + $status->save(); - return $status; - } + return $status; + } - public static function statusFetch($url) - { - return self::statusFirstOrFetch($url); - } + public static function statusFetch($url) + { + return self::statusFirstOrFetch($url); + } - public static function importNoteAttachment($data, Status $status) - { - if(self::verifyAttachments($data) == false) { - // \Log::info('importNoteAttachment::failedVerification.', [$data['id']]); - $status->viewType(); - return; - } - $attachments = isset($data['object']) ? $data['object']['attachment'] : $data['attachment']; - // peertube - // if(!$attachments) { - // $obj = isset($data['object']) ? $data['object'] : $data; - // $attachments = is_array($obj['url']) ? $obj['url'] : null; - // } - $user = $status->profile; - $storagePath = MediaPathService::get($user, 2); - $allowed = explode(',', config_cache('pixelfed.media_types')); + public static function importNoteAttachment($data, Status $status) + { + if(self::verifyAttachments($data) == false) { + // \Log::info('importNoteAttachment::failedVerification.', [$data['id']]); + $status->viewType(); + return; + } + $attachments = isset($data['object']) ? $data['object']['attachment'] : $data['attachment']; + // peertube + // if(!$attachments) { + // $obj = isset($data['object']) ? $data['object'] : $data; + // $attachments = is_array($obj['url']) ? $obj['url'] : null; + // } + $user = $status->profile; + $storagePath = MediaPathService::get($user, 2); + $allowed = explode(',', config_cache('pixelfed.media_types')); - foreach($attachments as $key => $media) { - $type = $media['mediaType']; - $url = $media['url']; - $valid = self::validateUrl($url); - if(in_array($type, $allowed) == false || $valid == false) { - continue; - } - $blurhash = isset($media['blurhash']) ? $media['blurhash'] : null; - $license = isset($media['license']) ? License::nameToId($media['license']) : null; - $caption = isset($media['name']) ? Purify::clean($media['name']) : null; + foreach($attachments as $key => $media) { + $type = $media['mediaType']; + $url = $media['url']; + $valid = self::validateUrl($url); + if(in_array($type, $allowed) == false || $valid == false) { + continue; + } + $blurhash = isset($media['blurhash']) ? $media['blurhash'] : null; + $license = isset($media['license']) ? License::nameToId($media['license']) : null; + $caption = isset($media['name']) ? Purify::clean($media['name']) : null; - $media = new Media(); - $media->blurhash = $blurhash; - $media->remote_media = true; - $media->status_id = $status->id; - $media->profile_id = $status->profile_id; - $media->user_id = null; - $media->media_path = $url; - $media->remote_url = $url; - $media->caption = $caption; - $media->order = $key + 1; - if($license) { - $media->license = $license; - } - $media->mime = $type; - $media->version = 3; - $media->save(); + $media = new Media(); + $media->blurhash = $blurhash; + $media->remote_media = true; + $media->status_id = $status->id; + $media->profile_id = $status->profile_id; + $media->user_id = null; + $media->media_path = $url; + $media->remote_url = $url; + $media->caption = $caption; + $media->order = $key + 1; + if($license) { + $media->license = $license; + } + $media->mime = $type; + $media->version = 3; + $media->save(); - if(config_cache('pixelfed.cloud_storage') == true) { - MediaStoragePipeline::dispatch($media); - } - } + if(config_cache('pixelfed.cloud_storage') == true) { + MediaStoragePipeline::dispatch($media); + } + } - $status->viewType(); - return; - } + $status->viewType(); + return; + } - public static function profileFirstOrNew($url) - { - $url = self::validateUrl($url); - if($url == false) { - return; - } + public static function profileFirstOrNew($url) + { + $url = self::validateUrl($url); + if($url == false) { + return; + } - $host = parse_url($url, PHP_URL_HOST); - $local = config('pixelfed.domain.app') == $host ? true : false; + $host = parse_url($url, PHP_URL_HOST); + $local = config('pixelfed.domain.app') == $host ? true : false; - if($local == true) { - $id = last(explode('/', $url)); - return Profile::whereNull('status') - ->whereNull('domain') - ->whereUsername($id) - ->firstOrFail(); - } + if($local == true) { + $id = last(explode('/', $url)); + return Profile::whereNull('status') + ->whereNull('domain') + ->whereUsername($id) + ->firstOrFail(); + } - if($profile = Profile::whereRemoteUrl($url)->first()) { - if($profile->last_fetched_at && $profile->last_fetched_at->lt(now()->subHours(24))) { - return self::profileUpdateOrCreate($url); - } - return $profile; - } + if($profile = Profile::whereRemoteUrl($url)->first()) { + if($profile->last_fetched_at && $profile->last_fetched_at->lt(now()->subHours(24))) { + return self::profileUpdateOrCreate($url); + } + return $profile; + } - return self::profileUpdateOrCreate($url); - } + return self::profileUpdateOrCreate($url); + } - public static function profileUpdateOrCreate($url) - { - $res = self::fetchProfileFromUrl($url); - if(!$res || isset($res['id']) == false) { - return; - } - $domain = parse_url($res['id'], PHP_URL_HOST); - if(!isset($res['preferredUsername']) && !isset($res['nickname'])) { - return; - } - $username = (string) Purify::clean($res['preferredUsername'] ?? $res['nickname']); - if(empty($username)) { - return; - } - $remoteUsername = $username; - $webfinger = "@{$username}@{$domain}"; + public static function profileUpdateOrCreate($url) + { + $res = self::fetchProfileFromUrl($url); + if(!$res || isset($res['id']) == false) { + return; + } + $domain = parse_url($res['id'], PHP_URL_HOST); + if(!isset($res['preferredUsername']) && !isset($res['nickname'])) { + return; + } + $username = (string) Purify::clean($res['preferredUsername'] ?? $res['nickname']); + if(empty($username)) { + return; + } + $remoteUsername = $username; + $webfinger = "@{$username}@{$domain}"; - if(!self::validateUrl($res['inbox'])) { - return; - } - if(!self::validateUrl($res['id'])) { - return; - } + if(!self::validateUrl($res['inbox'])) { + return; + } + if(!self::validateUrl($res['id'])) { + return; + } - $instance = Instance::updateOrCreate([ - 'domain' => $domain - ]); - if($instance->wasRecentlyCreated == true) { - \App\Jobs\InstancePipeline\FetchNodeinfoPipeline::dispatch($instance)->onQueue('low'); - } + $instance = Instance::updateOrCreate([ + 'domain' => $domain + ]); + if($instance->wasRecentlyCreated == true) { + \App\Jobs\InstancePipeline\FetchNodeinfoPipeline::dispatch($instance)->onQueue('low'); + } - $profile = Profile::updateOrCreate( - [ - 'domain' => strtolower($domain), - 'username' => Purify::clean($webfinger), - ], - [ - 'webfinger' => Purify::clean($webfinger), - 'key_id' => $res['publicKey']['id'], - 'remote_url' => $res['id'], - 'name' => isset($res['name']) ? Purify::clean($res['name']) : 'user', - 'bio' => isset($res['summary']) ? Purify::clean($res['summary']) : null, - 'sharedInbox' => isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null, - 'inbox_url' => $res['inbox'], - 'outbox_url' => isset($res['outbox']) ? $res['outbox'] : null, - 'public_key' => $res['publicKey']['publicKeyPem'], - ] - ); + $profile = Profile::updateOrCreate( + [ + 'domain' => strtolower($domain), + 'username' => Purify::clean($webfinger), + ], + [ + 'webfinger' => Purify::clean($webfinger), + 'key_id' => $res['publicKey']['id'], + 'remote_url' => $res['id'], + 'name' => isset($res['name']) ? Purify::clean($res['name']) : 'user', + 'bio' => isset($res['summary']) ? Purify::clean($res['summary']) : null, + 'sharedInbox' => isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null, + 'inbox_url' => $res['inbox'], + 'outbox_url' => isset($res['outbox']) ? $res['outbox'] : null, + 'public_key' => $res['publicKey']['publicKeyPem'], + ] + ); - if( $profile->last_fetched_at == null || - $profile->last_fetched_at->lt(now()->subHours(24)) - ) { - RemoteAvatarFetch::dispatch($profile); - } - $profile->last_fetched_at = now(); - $profile->save(); - return $profile; - } + if( $profile->last_fetched_at == null || + $profile->last_fetched_at->lt(now()->subHours(24)) + ) { + RemoteAvatarFetch::dispatch($profile); + } + $profile->last_fetched_at = now(); + $profile->save(); + return $profile; + } - public static function profileFetch($url) - { - return self::profileFirstOrNew($url); - } + public static function profileFetch($url) + { + return self::profileFirstOrNew($url); + } - public static function sendSignedObject($profile, $url, $body) - { - ActivityPubDeliveryService::queue() - ->from($profile) - ->to($url) - ->payload($body) - ->send(); - } + public static function sendSignedObject($profile, $url, $body) + { + ActivityPubDeliveryService::queue() + ->from($profile) + ->to($url) + ->payload($body) + ->send(); + } } From c61d0b915f8d5dd3339c0ed54d8d52ae01b2cada Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 14 Jul 2023 01:22:49 -0600 Subject: [PATCH 06/10] Update SearchApiV2Service, improve resolve query logic to better handle remote posts/profiles and local posts/profiles --- app/Services/SearchApiV2Service.php | 606 +++++++++++++++------------- 1 file changed, 322 insertions(+), 284 deletions(-) diff --git a/app/Services/SearchApiV2Service.php b/app/Services/SearchApiV2Service.php index 465bd98a4..90691f0bd 100644 --- a/app/Services/SearchApiV2Service.php +++ b/app/Services/SearchApiV2Service.php @@ -18,317 +18,355 @@ use App\Services\StatusService; class SearchApiV2Service { - private $query; - static $mastodonMode = false; + private $query; + static $mastodonMode = false; - public static function query($query, $mastodonMode = false) - { - self::$mastodonMode = $mastodonMode; - return (new self)->run($query); - } + public static function query($query, $mastodonMode = false) + { + self::$mastodonMode = $mastodonMode; + return (new self)->run($query); + } - protected function run($query) - { - $this->query = $query; - $q = urldecode($query->input('q')); + protected function run($query) + { + $this->query = $query; + $q = urldecode($query->input('q')); - if($query->has('resolve') && - ( Str::startsWith($q, 'https://') || - Str::substrCount($q, '@') >= 1) - ) { - return $this->resolveQuery(); - } + if($query->has('resolve') && + ( Str::startsWith($q, 'https://') || + Str::substrCount($q, '@') >= 1) + ) { + return $this->resolveQuery(); + } - if($query->has('type')) { - switch ($query->input('type')) { - case 'accounts': - return [ - 'accounts' => $this->accounts(), - 'hashtags' => [], - 'statuses' => [] - ]; - break; - case 'hashtags': - return [ - 'accounts' => [], - 'hashtags' => $this->hashtags(), - 'statuses' => [] - ]; - break; - case 'statuses': - return [ - 'accounts' => [], - 'hashtags' => [], - 'statuses' => $this->statuses() - ]; - break; - } - } + if($query->has('type')) { + switch ($query->input('type')) { + case 'accounts': + return [ + 'accounts' => $this->accounts(), + 'hashtags' => [], + 'statuses' => [] + ]; + break; + case 'hashtags': + return [ + 'accounts' => [], + 'hashtags' => $this->hashtags(), + 'statuses' => [] + ]; + break; + case 'statuses': + return [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => $this->statuses() + ]; + break; + } + } - if($query->has('account_id')) { - return [ - 'accounts' => [], - 'hashtags' => [], - 'statuses' => $this->statusesById() - ]; - } + if($query->has('account_id')) { + return [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => $this->statusesById() + ]; + } - return [ - 'accounts' => $this->accounts(), - 'hashtags' => $this->hashtags(), - 'statuses' => $this->statuses() - ]; - } + return [ + 'accounts' => $this->accounts(), + 'hashtags' => $this->hashtags(), + 'statuses' => $this->statuses() + ]; + } - protected function accounts($initalQuery = false) - { - $mastodonMode = self::$mastodonMode; - $user = request()->user(); - $limit = $this->query->input('limit') ?? 20; - $offset = $this->query->input('offset') ?? 0; - $rawQuery = $initalQuery ? $initalQuery : $this->query->input('q'); - $query = $rawQuery . '%'; - $webfingerQuery = $query; - if(Str::substrCount($rawQuery, '@') == 1 && substr($rawQuery, 0, 1) !== '@') { - $query = '@' . $query; - } - if(substr($webfingerQuery, 0, 1) !== '@') { - $webfingerQuery = '@' . $webfingerQuery; - } - $banned = InstanceService::getBannedDomains(); - $operator = config('database.default') === 'pgsql' ? 'ilike' : 'like'; - $results = Profile::select('username', 'id', 'followers_count', 'domain') - ->where('username', $operator, $query) - ->orWhere('webfinger', $operator, $webfingerQuery) - ->orderByDesc('profiles.followers_count') - ->offset($offset) - ->limit($limit) - ->get() - ->filter(function($profile) use ($banned) { - return in_array($profile->domain, $banned) == false; - }) - ->map(function($res) use($mastodonMode) { - return $mastodonMode ? - AccountService::getMastodon($res['id']) : - AccountService::get($res['id']); - }) - ->filter(function($account) { - return $account && isset($account['id']); - }) - ->values(); + protected function accounts($initalQuery = false) + { + $mastodonMode = self::$mastodonMode; + $user = request()->user(); + $limit = $this->query->input('limit') ?? 20; + $offset = $this->query->input('offset') ?? 0; + $rawQuery = $initalQuery ? $initalQuery : $this->query->input('q'); + $query = $rawQuery . '%'; + $webfingerQuery = $query; + if(Str::substrCount($rawQuery, '@') == 1 && substr($rawQuery, 0, 1) !== '@') { + $query = '@' . $query; + } + if(substr($webfingerQuery, 0, 1) !== '@') { + $webfingerQuery = '@' . $webfingerQuery; + } + $banned = InstanceService::getBannedDomains(); + $operator = config('database.default') === 'pgsql' ? 'ilike' : 'like'; + $results = Profile::select('username', 'id', 'followers_count', 'domain') + ->where('username', $operator, $query) + ->orWhere('webfinger', $operator, $webfingerQuery) + ->orderByDesc('profiles.followers_count') + ->offset($offset) + ->limit($limit) + ->get() + ->filter(function($profile) use ($banned) { + return in_array($profile->domain, $banned) == false; + }) + ->map(function($res) use($mastodonMode) { + return $mastodonMode ? + AccountService::getMastodon($res['id']) : + AccountService::get($res['id']); + }) + ->filter(function($account) { + return $account && isset($account['id']); + }) + ->values(); - return $results; - } + return $results; + } - protected function hashtags() - { - $mastodonMode = self::$mastodonMode; - $q = $this->query->input('q'); - $limit = $this->query->input('limit') ?? 20; - $offset = $this->query->input('offset') ?? 0; - $query = Str::startsWith($q, '#') ? '%' . substr($q, 1) . '%' : '%' . $q . '%'; - $operator = config('database.default') === 'pgsql' ? 'ilike' : 'like'; - return Hashtag::where('name', $operator, $query) - ->orWhere('slug', $operator, $query) - ->where(function($q) { - return $q->where('can_search', true) - ->orWhereNull('can_search'); - }) - ->orderByDesc('cached_count') - ->offset($offset) - ->limit($limit) - ->get() - ->map(function($tag) use($mastodonMode) { - $res = [ - 'name' => $tag->name, - 'url' => $tag->url() - ]; + protected function hashtags() + { + $mastodonMode = self::$mastodonMode; + $q = $this->query->input('q'); + $limit = $this->query->input('limit') ?? 20; + $offset = $this->query->input('offset') ?? 0; + $query = Str::startsWith($q, '#') ? '%' . substr($q, 1) . '%' : '%' . $q . '%'; + $operator = config('database.default') === 'pgsql' ? 'ilike' : 'like'; + return Hashtag::where('name', $operator, $query) + ->orWhere('slug', $operator, $query) + ->where(function($q) { + return $q->where('can_search', true) + ->orWhereNull('can_search'); + }) + ->orderByDesc('cached_count') + ->offset($offset) + ->limit($limit) + ->get() + ->map(function($tag) use($mastodonMode) { + $res = [ + 'name' => $tag->name, + 'url' => $tag->url() + ]; - if(!$mastodonMode) { - $res['history'] = []; - $res['count'] = HashtagService::count($tag->id); - } + if(!$mastodonMode) { + $res['history'] = []; + $res['count'] = HashtagService::count($tag->id); + } - return $res; - }); - } + return $res; + }); + } - protected function statuses() - { - // Removed until we provide more relevent sorting/results - return []; - } + protected function statuses() + { + // Removed until we provide more relevent sorting/results + return []; + } - protected function statusesById() - { - // Removed until we provide more relevent sorting/results - return []; - } + protected function statusesById() + { + // Removed until we provide more relevent sorting/results + return []; + } - protected function resolveQuery() - { - $default = [ - 'accounts' => [], - 'hashtags' => [], - 'statuses' => [], - ]; - $mastodonMode = self::$mastodonMode; - $query = urldecode($this->query->input('q')); - if(substr($query, 0, 1) === '@' && !Str::contains($query, '.')) { - $default['accounts'] = $this->accounts(substr($query, 1)); - return $default; - } - if(Helpers::validateLocalUrl($query)) { - if(Str::contains($query, '/p/')) { - return $this->resolveLocalStatus(); - } else { - return $this->resolveLocalProfile(); - } - } else { - if(!Helpers::validateUrl($query) && strpos($query, '@') == -1) { - return $default; - } + protected function resolveQuery() + { + $default = [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => [], + ]; + $mastodonMode = self::$mastodonMode; + $query = urldecode($this->query->input('q')); + if(substr($query, 0, 1) === '@' && !Str::contains($query, '.')) { + $default['accounts'] = $this->accounts(substr($query, 1)); + return $default; + } + if(Helpers::validateLocalUrl($query)) { + if(Str::contains($query, '/p/') || Str::contains($query, 'i/web/post/')) { + return $this->resolveLocalStatus(); + } else if(Str::contains($query, 'i/web/profile/')) { + return $this->resolveLocalProfileId(); + } else { + return $this->resolveLocalProfile(); + } + } else { + if(!Helpers::validateUrl($query) && strpos($query, '@') == -1) { + return $default; + } - if(!Str::startsWith($query, 'http') && Str::substrCount($query, '@') == 1 && strpos($query, '@') !== 0) { - try { - $res = WebfingerService::lookup('@' . $query, $mastodonMode); - } catch (\Exception $e) { - return $default; - } - if($res && isset($res['id'])) { - $default['accounts'][] = $res; - return $default; - } else { - return $default; - } - } + if(!Str::startsWith($query, 'http') && Str::substrCount($query, '@') == 1 && strpos($query, '@') !== 0) { + try { + $res = WebfingerService::lookup('@' . $query, $mastodonMode); + } catch (\Exception $e) { + return $default; + } + if($res && isset($res['id'])) { + $default['accounts'][] = $res; + return $default; + } else { + return $default; + } + } - if(Str::substrCount($query, '@') == 2) { - try { - $res = WebfingerService::lookup($query, $mastodonMode); - } catch (\Exception $e) { - return $default; - } - if($res && isset($res['id'])) { - $default['accounts'][] = $res; - return $default; - } else { - return $default; - } - } + if(Str::substrCount($query, '@') == 2) { + try { + $res = WebfingerService::lookup($query, $mastodonMode); + } catch (\Exception $e) { + return $default; + } + if($res && isset($res['id'])) { + $default['accounts'][] = $res; + return $default; + } else { + return $default; + } + } - try { - $res = ActivityPubFetchService::get($query); - $banned = InstanceService::getBannedDomains(); - if($res) { - $json = json_decode($res, true); + if($sid = Status::whereUri($query)->first()) { + $s = StatusService::get($sid->id, false); + if(in_array($s['visibility'], ['public', 'unlisted'])) { + $default['statuses'][] = $s; + return $default; + } + } - if(!$json || !isset($json['@context']) || !isset($json['type']) || !in_array($json['type'], ['Note', 'Person'])) { - return [ - 'accounts' => [], - 'hashtags' => [], - 'statuses' => [], - ]; - } + try { + $res = ActivityPubFetchService::get($query); + $banned = InstanceService::getBannedDomains(); + if($res) { + $json = json_decode($res, true); - switch($json['type']) { - case 'Note': - $obj = Helpers::statusFetch($query); - if(!$obj || !isset($obj['id'])) { - return $default; - } - $note = $mastodonMode ? - StatusService::getMastodon($obj['id']) : - StatusService::get($obj['id']); - if(!$note) { - return $default; - } - $default['statuses'][] = $note; - return $default; - break; + if(!$json || !isset($json['@context']) || !isset($json['type']) || !in_array($json['type'], ['Note', 'Person'])) { + return [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => [], + ]; + } - case 'Person': - $obj = Helpers::profileFetch($query); - if(!$obj) { - return $default; - } - if(in_array($obj['domain'], $banned)) { - return $default; - } - $default['accounts'][] = $mastodonMode ? - AccountService::getMastodon($obj['id']) : - AccountService::get($obj['id']); - return $default; - break; + switch($json['type']) { + case 'Note': + $obj = Helpers::statusFetch($query); + if(!$obj || !isset($obj['id'])) { + return $default; + } + $note = $mastodonMode ? + StatusService::getMastodon($obj['id'], false) : + StatusService::get($obj['id'], false); + if(!$note) { + return $default; + } + if(!isset($note['visibility']) || !in_array($note['visibility'], ['public', 'unlisted'])) { + return $default; + } + $default['statuses'][] = $note; + return $default; + break; - default: - return [ - 'accounts' => [], - 'hashtags' => [], - 'statuses' => [], - ]; - break; - } - } - } catch (\Exception $e) { - return [ - 'accounts' => [], - 'hashtags' => [], - 'statuses' => [], - ]; - } + case 'Person': + $obj = Helpers::profileFetch($query); + if(!$obj) { + return $default; + } + if(in_array($obj['domain'], $banned)) { + return $default; + } + $default['accounts'][] = $mastodonMode ? + AccountService::getMastodon($obj['id'], true) : + AccountService::get($obj['id'], true); + return $default; + break; - return $default; - } - } + default: + return [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => [], + ]; + break; + } + } + } catch (\Exception $e) { + return [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => [], + ]; + } - protected function resolveLocalStatus() - { - $query = urldecode($this->query->input('q')); - $query = last(explode('/', $query)); - $status = StatusService::getMastodon($query); - if(!$status) { - return [ - 'accounts' => [], - 'hashtags' => [], - 'statuses' => [] - ]; - } + return $default; + } + } - $res = [ - 'accounts' => [], - 'hashtags' => [], - 'statuses' => [$status] - ]; + protected function resolveLocalStatus() + { + $query = urldecode($this->query->input('q')); + $query = last(explode('/', parse_url($query, PHP_URL_PATH))); + $status = StatusService::getMastodon($query, false); + if(!$status || !in_array($status['visibility'], ['public', 'unlisted'])) { + return [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => [] + ]; + } - return $res; - } + $res = [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => [$status] + ]; - protected function resolveLocalProfile() - { - $query = urldecode($this->query->input('q')); - $query = last(explode('/', $query)); - $profile = Profile::whereNull('status') - ->whereNull('domain') - ->whereUsername($query) - ->first(); + return $res; + } - if(!$profile) { - return [ - 'accounts' => [], - 'hashtags' => [], - 'statuses' => [] - ]; - } + protected function resolveLocalProfile() + { + $query = urldecode($this->query->input('q')); + $query = last(explode('/', parse_url($query, PHP_URL_PATH))); + $profile = Profile::whereNull('status') + ->whereNull('domain') + ->whereUsername($query) + ->first(); - $fractal = new Fractal\Manager(); - $fractal->setSerializer(new ArraySerializer()); - $resource = new Fractal\Resource\Item($profile, new AccountTransformer()); - return [ - 'accounts' => $fractal->createData($resource)->toArray(), - 'hashtags' => [], - 'statuses' => [] - ]; - } + if(!$profile) { + return [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => [] + ]; + } + + $fractal = new Fractal\Manager(); + $fractal->setSerializer(new ArraySerializer()); + $resource = new Fractal\Resource\Item($profile, new AccountTransformer()); + return [ + 'accounts' => [$fractal->createData($resource)->toArray()], + 'hashtags' => [], + 'statuses' => [] + ]; + } + + protected function resolveLocalProfileId() + { + $query = urldecode($this->query->input('q')); + $query = last(explode('/', parse_url($query, PHP_URL_PATH))); + $profile = Profile::whereNull('status') + ->find($query); + + if(!$profile) { + return [ + 'accounts' => [], + 'hashtags' => [], + 'statuses' => [] + ]; + } + + $fractal = new Fractal\Manager(); + $fractal->setSerializer(new ArraySerializer()); + $resource = new Fractal\Resource\Item($profile, new AccountTransformer()); + return [ + 'accounts' => [$fractal->createData($resource)->toArray()], + 'hashtags' => [], + 'statuses' => [] + ]; + } } From 0b5157675f30fff8ca9bf3e3a145efae845d7e13 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 14 Jul 2023 01:38:36 -0600 Subject: [PATCH 07/10] Update FollowPipeline, improve follower/following count calculation --- app/Jobs/FollowPipeline/FollowPipeline.php | 134 +++++++++------------ 1 file changed, 57 insertions(+), 77 deletions(-) diff --git a/app/Jobs/FollowPipeline/FollowPipeline.php b/app/Jobs/FollowPipeline/FollowPipeline.php index 6db85fa6a..225334304 100644 --- a/app/Jobs/FollowPipeline/FollowPipeline.php +++ b/app/Jobs/FollowPipeline/FollowPipeline.php @@ -17,91 +17,71 @@ use App\Services\FollowerService; class FollowPipeline implements ShouldQueue { - use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $follower; + protected $follower; - /** - * Delete the job if its models no longer exist. - * - * @var bool - */ - public $deleteWhenMissingModels = true; - - /** - * Create a new job instance. - * - * @return void - */ - public function __construct($follower) - { - $this->follower = $follower; - } + /** + * Delete the job if its models no longer exist. + * + * @var bool + */ + public $deleteWhenMissingModels = true; - /** - * Execute the job. - * - * @return void - */ - public function handle() - { - $follower = $this->follower; - $actor = $follower->actor; - $target = $follower->target; + /** + * Create a new job instance. + * + * @return void + */ + public function __construct($follower) + { + $this->follower = $follower; + } - if(!$actor || !$target) { - return; - } + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $follower = $this->follower; + $actor = $follower->actor; + $target = $follower->target; - Cache::forget('profile:following:' . $actor->id); - Cache::forget('profile:following:' . $target->id); + if(!$actor || !$target) { + return; + } - FollowerService::add($actor->id, $target->id); + if($target->domain || !$target->private_key) { + return; + } - $actorProfileSync = Cache::get(FollowerService::FOLLOWING_SYNC_KEY . $actor->id); - if(!$actorProfileSync) { - FollowServiceWarmCache::dispatch($actor->id)->onQueue('low'); - } else { - if($actor->following_count) { - $actor->increment('following_count'); - } else { - $count = Follower::whereProfileId($actor->id)->count(); - $actor->following_count = $count; - $actor->save(); - } - Cache::put(FollowerService::FOLLOWING_SYNC_KEY . $actor->id, 1, 604800); - AccountService::del($actor->id); - } + Cache::forget('profile:following:' . $actor->id); + Cache::forget('profile:following:' . $target->id); - $targetProfileSync = Cache::get(FollowerService::FOLLOWERS_SYNC_KEY . $target->id); - if(!$targetProfileSync) { - FollowServiceWarmCache::dispatch($target->id)->onQueue('low'); - } else { - if($target->followers_count) { - $target->increment('followers_count'); - } else { - $count = Follower::whereFollowingId($target->id)->count(); - $target->followers_count = $count; - $target->save(); - } - Cache::put(FollowerService::FOLLOWERS_SYNC_KEY . $target->id, 1, 604800); - AccountService::del($target->id); - } + FollowerService::add($actor->id, $target->id); - if($target->domain || !$target->private_key) { - return; - } + $count = Follower::whereProfileId($actor->id)->count(); + $actor->following_count = $count; + $actor->save(); + AccountService::del($actor->id); - try { - $notification = new Notification(); - $notification->profile_id = $target->id; - $notification->actor_id = $actor->id; - $notification->action = 'follow'; - $notification->item_id = $target->id; - $notification->item_type = "App\Profile"; - $notification->save(); - } catch (Exception $e) { - Log::error($e); - } - } + $count = Follower::whereFollowingId($target->id)->count(); + $target->followers_count = $count; + $target->save(); + AccountService::del($target->id); + + try { + $notification = new Notification(); + $notification->profile_id = $target->id; + $notification->actor_id = $actor->id; + $notification->action = 'follow'; + $notification->item_id = $target->id; + $notification->item_type = "App\Profile"; + $notification->save(); + } catch (Exception $e) { + Log::error($e); + } + } } From ba7551d8a9f9831093a82f464d1dcbc8e70bc5d9 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 14 Jul 2023 01:46:51 -0600 Subject: [PATCH 08/10] Update TransformImports command, increment status_count on profile model --- app/Console/Commands/TransformImports.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/Console/Commands/TransformImports.php b/app/Console/Commands/TransformImports.php index cd63985ac..b88401178 100644 --- a/app/Console/Commands/TransformImports.php +++ b/app/Console/Commands/TransformImports.php @@ -9,6 +9,7 @@ use App\Media; use App\Profile; use App\Status; use Storage; +use App\Services\AccountService; use App\Services\MediaPathService; use Illuminate\Support\Str; use App\Util\Lexer\Autolink; @@ -38,7 +39,7 @@ class TransformImports extends Command return; } - $ips = ImportPost::whereNull('status_id')->where('skip_missing_media', '!=', true)->take(200)->get(); + $ips = ImportPost::whereNull('status_id')->where('skip_missing_media', '!=', true)->take(500)->get(); if(!$ips->count()) { return; @@ -135,6 +136,11 @@ class TransformImports extends Command $ip->creation_id = $idk['incr']; $ip->save(); + $profile->status_count = $profile->status_count + 1; + $profile->save(); + + AccountService::del($profile->id); + ImportService::clearAttempts($profile->id); ImportService::getPostCount($profile->id, true); } From 84669ac6140428f111f6705cc413c6ea4439c330 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 14 Jul 2023 01:48:34 -0600 Subject: [PATCH 09/10] Update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49d6c00a4..28aa63658 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,13 @@ - Update ComposeModal.vue, fix scroll issue and dont hide scrollbar ([2d959fb3](https://github.com/pixelfed/pixelfed/commit/2d959fb3)) - Update AccountImport, add select first 100 posts button ([625a76a5](https://github.com/pixelfed/pixelfed/commit/625a76a5)) - Update ApiV1Controller, add include_reblogs attribute to home timeline ([37fd0342](https://github.com/pixelfed/pixelfed/commit/37fd0342)) +- Update rate limits, fixes #4537 ([1cc6274a](https://github.com/pixelfed/pixelfed/commit/1cc6274a)) +- Update Services, use zpopmin on predis ([4b2c66f5](https://github.com/pixelfed/pixelfed/commit/4b2c66f5)) +- Update Inbox, allow storing Create->Note activities without any local followers, disabled by default ([9fa6b3f7](https://github.com/pixelfed/pixelfed/commit/9fa6b3f7)) +- Update AP Helpers, preserve admin unlisted state before adding to NetworkTimelineService ([0704c7e0](https://github.com/pixelfed/pixelfed/commit/0704c7e0)) +- Update SearchApiV2Service, improve resolve query logic to better handle remote posts/profiles and local posts/profiles ([c61d0b91](https://github.com/pixelfed/pixelfed/commit/c61d0b91)) +- Update FollowPipeline, improve follower/following count calculation ([0b515767](https://github.com/pixelfed/pixelfed/commit/0b515767)) +- Update TransformImports command, increment status_count on profile model ([ba7551d8](https://github.com/pixelfed/pixelfed/commit/ba7551d8)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.8 (2023-05-29)](https://github.com/pixelfed/pixelfed/compare/v0.11.7...v0.11.8) From a00a520bf394052ed9d063ec9a6765400966dfbd Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 14 Jul 2023 01:50:43 -0600 Subject: [PATCH 10/10] Update composer deps --- composer.lock | 75 ++++++++++++++++++++++++++------------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/composer.lock b/composer.lock index e28376f20..74c453c67 100644 --- a/composer.lock +++ b/composer.lock @@ -62,16 +62,16 @@ }, { "name": "aws/aws-sdk-php", - "version": "3.275.5", + "version": "3.275.7", "source": { "type": "git", "url": "https://github.com/aws/aws-sdk-php.git", - "reference": "d46961b82e857f77059c0c78160719ecb26f6cc6" + "reference": "54dcef3349c81b46c0f5f6e54b5f9bfb5db19903" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/d46961b82e857f77059c0c78160719ecb26f6cc6", - "reference": "d46961b82e857f77059c0c78160719ecb26f6cc6", + "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/54dcef3349c81b46c0f5f6e54b5f9bfb5db19903", + "reference": "54dcef3349c81b46c0f5f6e54b5f9bfb5db19903", "shasum": "" }, "require": { @@ -151,9 +151,9 @@ "support": { "forum": "https://forums.aws.amazon.com/forum.jspa?forumID=80", "issues": "https://github.com/aws/aws-sdk-php/issues", - "source": "https://github.com/aws/aws-sdk-php/tree/3.275.5" + "source": "https://github.com/aws/aws-sdk-php/tree/3.275.7" }, - "time": "2023-07-07T18:20:11+00:00" + "time": "2023-07-13T18:21:04+00:00" }, { "name": "bacon/bacon-qr-code", @@ -2357,16 +2357,16 @@ }, { "name": "laravel/framework", - "version": "v10.14.1", + "version": "v10.15.0", "source": { "type": "git", "url": "https://github.com/laravel/framework.git", - "reference": "6f89a2b74b232d8bf2e1d9ed87e311841263dfcb" + "reference": "c7599dc92e04532824bafbd226c2936ce6a905b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/framework/zipball/6f89a2b74b232d8bf2e1d9ed87e311841263dfcb", - "reference": "6f89a2b74b232d8bf2e1d9ed87e311841263dfcb", + "url": "https://api.github.com/repos/laravel/framework/zipball/c7599dc92e04532824bafbd226c2936ce6a905b8", + "reference": "c7599dc92e04532824bafbd226c2936ce6a905b8", "shasum": "" }, "require": { @@ -2553,7 +2553,7 @@ "issues": "https://github.com/laravel/framework/issues", "source": "https://github.com/laravel/framework" }, - "time": "2023-06-28T14:25:16+00:00" + "time": "2023-07-11T13:43:52+00:00" }, { "name": "laravel/helpers", @@ -2613,16 +2613,16 @@ }, { "name": "laravel/horizon", - "version": "v5.17.0", + "version": "v5.18.0", "source": { "type": "git", "url": "https://github.com/laravel/horizon.git", - "reference": "569c7154033679a1ca05b43bfa640cc60aa3b37b" + "reference": "b14498a09af826035e46ae8d6b013d0ec849bdb7" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/horizon/zipball/569c7154033679a1ca05b43bfa640cc60aa3b37b", - "reference": "569c7154033679a1ca05b43bfa640cc60aa3b37b", + "url": "https://api.github.com/repos/laravel/horizon/zipball/b14498a09af826035e46ae8d6b013d0ec849bdb7", + "reference": "b14498a09af826035e46ae8d6b013d0ec849bdb7", "shasum": "" }, "require": { @@ -2685,9 +2685,9 @@ ], "support": { "issues": "https://github.com/laravel/horizon/issues", - "source": "https://github.com/laravel/horizon/tree/v5.17.0" + "source": "https://github.com/laravel/horizon/tree/v5.18.0" }, - "time": "2023-06-13T20:49:30+00:00" + "time": "2023-06-30T15:11:51+00:00" }, { "name": "laravel/passport", @@ -6651,23 +6651,24 @@ }, { "name": "react/promise", - "version": "v2.10.0", + "version": "v3.0.0", "source": { "type": "git", "url": "https://github.com/reactphp/promise.git", - "reference": "f913fb8cceba1e6644b7b90c4bfb678ed8a3ef38" + "reference": "c86753c76fd3be465d93b308f18d189f01a22be4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/reactphp/promise/zipball/f913fb8cceba1e6644b7b90c4bfb678ed8a3ef38", - "reference": "f913fb8cceba1e6644b7b90c4bfb678ed8a3ef38", + "url": "https://api.github.com/repos/reactphp/promise/zipball/c86753c76fd3be465d93b308f18d189f01a22be4", + "reference": "c86753c76fd3be465d93b308f18d189f01a22be4", "shasum": "" }, "require": { - "php": ">=5.4.0" + "php": ">=7.1.0" }, "require-dev": { - "phpunit/phpunit": "^9.5 || ^5.7 || ^4.8.36" + "phpstan/phpstan": "1.10.20 || 1.4.10", + "phpunit/phpunit": "^9.5 || ^7.5" }, "type": "library", "autoload": { @@ -6711,7 +6712,7 @@ ], "support": { "issues": "https://github.com/reactphp/promise/issues", - "source": "https://github.com/reactphp/promise/tree/v2.10.0" + "source": "https://github.com/reactphp/promise/tree/v3.0.0" }, "funding": [ { @@ -6719,7 +6720,7 @@ "type": "open_collective" } ], - "time": "2023-05-02T15:15:43+00:00" + "time": "2023-07-11T16:12:49+00:00" }, { "name": "react/socket", @@ -10920,16 +10921,16 @@ }, { "name": "filp/whoops", - "version": "2.15.2", + "version": "2.15.3", "source": { "type": "git", "url": "https://github.com/filp/whoops.git", - "reference": "aac9304c5ed61bf7b1b7a6064bf9806ab842ce73" + "reference": "c83e88a30524f9360b11f585f71e6b17313b7187" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/filp/whoops/zipball/aac9304c5ed61bf7b1b7a6064bf9806ab842ce73", - "reference": "aac9304c5ed61bf7b1b7a6064bf9806ab842ce73", + "url": "https://api.github.com/repos/filp/whoops/zipball/c83e88a30524f9360b11f585f71e6b17313b7187", + "reference": "c83e88a30524f9360b11f585f71e6b17313b7187", "shasum": "" }, "require": { @@ -10979,7 +10980,7 @@ ], "support": { "issues": "https://github.com/filp/whoops/issues", - "source": "https://github.com/filp/whoops/tree/2.15.2" + "source": "https://github.com/filp/whoops/tree/2.15.3" }, "funding": [ { @@ -10987,7 +10988,7 @@ "type": "github" } ], - "time": "2023-04-12T12:00:00+00:00" + "time": "2023-07-13T12:00:00+00:00" }, { "name": "hamcrest/hamcrest-php", @@ -11101,16 +11102,16 @@ }, { "name": "laravel/telescope", - "version": "v4.15.0", + "version": "v4.15.2", "source": { "type": "git", "url": "https://github.com/laravel/telescope.git", - "reference": "572a19b4c9b09295848de9a2352737a756a0fb05" + "reference": "5d74ae4c9f269b756d7877ad1527770c59846e14" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/laravel/telescope/zipball/572a19b4c9b09295848de9a2352737a756a0fb05", - "reference": "572a19b4c9b09295848de9a2352737a756a0fb05", + "url": "https://api.github.com/repos/laravel/telescope/zipball/5d74ae4c9f269b756d7877ad1527770c59846e14", + "reference": "5d74ae4c9f269b756d7877ad1527770c59846e14", "shasum": "" }, "require": { @@ -11166,9 +11167,9 @@ ], "support": { "issues": "https://github.com/laravel/telescope/issues", - "source": "https://github.com/laravel/telescope/tree/v4.15.0" + "source": "https://github.com/laravel/telescope/tree/v4.15.2" }, - "time": "2023-06-08T13:57:22+00:00" + "time": "2023-07-13T20:06:27+00:00" }, { "name": "mockery/mockery",