From b4bd0400c20e86bcf86debeae44fb7dbd029beda Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 2 Jan 2022 21:30:02 -0700 Subject: [PATCH 1/3] Store remote avatars locally if S3 not enabled --- app/Jobs/AvatarPipeline/RemoteAvatarFetch.php | 11 ++++++++--- app/Services/MediaStorageService.php | 13 +++++++------ app/Util/ActivityPub/Helpers.php | 8 ++------ config/federation.php | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php index 74dacaf20..7ffeb8a4c 100644 --- a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php +++ b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php @@ -32,6 +32,13 @@ class RemoteAvatarFetch implements ShouldQueue */ public $deleteWhenMissingModels = true; + /** + * The number of times the job may be attempted. + * + * @var int + */ + public $tries = 1; + /** * Create a new job instance. * @@ -99,9 +106,7 @@ class RemoteAvatarFetch implements ShouldQueue $avatar->remote_url = $icon['url']; $avatar->save(); - if(config_cache('pixelfed.cloud_storage')) { - MediaStorageService::avatar($avatar); - } + MediaStorageService::avatar($avatar, config_cache('pixelfed.cloud_storage') == false); return 1; } diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index 89d7ccdaa..5d58ee077 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -27,9 +27,9 @@ class MediaStorageService { return; } - public static function avatar($avatar) + public static function avatar($avatar, $local = false) { - return (new self())->fetchAvatar($avatar); + return (new self())->fetchAvatar($avatar, $local); } public static function head($url) @@ -177,11 +177,12 @@ class MediaStorageService { unlink($tmpName); } - protected function fetchAvatar($avatar) + protected function fetchAvatar($avatar, $local = false) { $url = $avatar->remote_url; + $driver = $local ? 'local' : config('filesystems.cloud'); - if($url == null || Helpers::validateUrl($url) == false) { + if(empty($url) || Helpers::validateUrl($url) == false) { return; } @@ -220,7 +221,7 @@ class MediaStorageService { return; } - $base = 'cache/avatars/' . $avatar->profile_id; + $base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id; $ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png'; $path = Str::random(20) . '_avatar.' . $ext; $tmpBase = storage_path('app/remcache/'); @@ -229,7 +230,7 @@ class MediaStorageService { $data = file_get_contents($url, false, null, 0, $head['length']); file_put_contents($tmpName, $data); - $disk = Storage::disk(config('filesystems.cloud')); + $disk = Storage::disk($driver); $file = $disk->putFileAs($base, new File($tmpName), $path, 'public'); $permalink = $disk->url($file); diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 907097f05..c2a88ed5c 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -586,9 +586,7 @@ class Helpers { $profile->webfinger = Purify::clean($webfinger); $profile->last_fetched_at = now(); $profile->save(); - if(config_cache('pixelfed.cloud_storage') == true) { - RemoteAvatarFetch::dispatch($profile); - } + RemoteAvatarFetch::dispatch($profile); return $profile; }); }); @@ -603,9 +601,7 @@ class Helpers { $profile->sharedInbox = isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) && Helpers::validateUrl($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null; $profile->save(); } - if(config_cache('pixelfed.cloud_storage') == true) { - RemoteAvatarFetch::dispatch($profile); - } + RemoteAvatarFetch::dispatch($profile); } return $profile; }); diff --git a/config/federation.php b/config/federation.php index 4f8a02389..ce2a5770e 100644 --- a/config/federation.php +++ b/config/federation.php @@ -33,7 +33,7 @@ return [ ], 'avatars' => [ - 'store_local' => false + 'store_local' => env('REMOTE_AVATARS', true), ], 'nodeinfo' => [ From 63e44d7c50e334afc2eb8fbfeb978f29da8b45db Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 2 Jan 2022 21:32:24 -0700 Subject: [PATCH 2/3] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb2eb5d1b..d26ea6643 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Added StatusMentionService, fixes #3026. ([e5387d67](https://github.com/pixelfed/pixelfed/commit/e5387d67)) - Cloud Backups, a command to store backups on S3 or compatible filesystems. [#3037](https://github.com/pixelfed/pixelfed/pull/3037) ([3515a98e](https://github.com/pixelfed/pixelfed/commit/3515a98e)) - Web UI Localizations + Crowdin integration. ([f7d9b40b](https://github.com/pixelfed/pixelfed/commit/f7d9b40b)) ([7ff120c9](https://github.com/pixelfed/pixelfed/commit/7ff120c9)) +- Store remote avatars locally if S3 not enabled. ([b4bd0400](https://github.com/pixelfed/pixelfed/commit/b4bd0400)) ### Updated - Updated NotificationService, fix 500 bug. ([4a609dc3](https://github.com/pixelfed/pixelfed/commit/4a609dc3)) From 41cc741be6d1668894f0b22bbbbb7a8ffbc8f66d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Sun, 2 Jan 2022 21:34:15 -0700 Subject: [PATCH 3/3] Add avatar:sync command --- app/Console/Commands/AvatarSync.php | 219 ++++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 app/Console/Commands/AvatarSync.php diff --git a/app/Console/Commands/AvatarSync.php b/app/Console/Commands/AvatarSync.php new file mode 100644 index 000000000..57d984747 --- /dev/null +++ b/app/Console/Commands/AvatarSync.php @@ -0,0 +1,219 @@ +info('Welcome to the avatar sync manager'); + + $actions = [ + 'Analyze', + 'Full Analyze', + 'Fetch - Fetch missing remote avatars', + 'Fix - Fix remote accounts without avatar record', + 'Sync - Store latest remote avatars', + ]; + + $name = $this->choice( + 'Select an action', + $actions, + 0, + 1, + false + ); + + $this->info('Selected: ' . $name); + + switch($name) { + case $actions[0]: + $this->analyze(); + break; + + case $actions[1]: + $this->fullAnalyze(); + break; + + case $actions[2]: + $this->fetch(); + break; + + case $actions[3]: + $this->fix(); + break; + + case $actions[4]: + $this->sync(); + break; + } + return Command::SUCCESS; + } + + protected function incr($name) + { + switch($name) { + case 'found': + $this->found = $this->found + 1; + break; + + case 'notFetched': + $this->notFetched = $this->notFetched + 1; + break; + + case 'fixed': + $this->fixed++; + break; + } + } + + protected function analyze() + { + $count = Avatar::whereIsRemote(true)->whereNull('cdn_url')->count(); + $this->info('Found ' . $count . ' profiles with blank avatars.'); + $this->line(' '); + $this->comment('We suggest running php artisan avatars:sync again and selecting the sync option'); + $this->line(' '); + } + + protected function fullAnalyze() + { + $count = Profile::count(); + $bar = $this->output->createProgressBar($count); + $bar->start(); + + Profile::chunk(5000, function($profiles) use ($bar) { + foreach($profiles as $profile) { + if($profile->domain == null) { + $bar->advance(); + continue; + } + $avatar = Avatar::whereProfileId($profile->id)->first(); + if(!$avatar || $avatar->cdn_url == null) { + $this->incr('notFetched'); + } + $this->incr('found'); + $bar->advance(); + } + }); + + $this->line(' '); + $this->line(' '); + $this->info('Found ' . $this->found . ' remote accounts'); + $this->info('Found ' . $this->notFetched . ' remote avatars to fetch'); + } + + protected function fetch() + { + $this->info('Fetching ....'); + Avatar::whereIsRemote(true) + ->whereNull('cdn_url') + // ->with('profile') + ->chunk(10, function($avatars) { + foreach($avatars as $avatar) { + if(!$avatar || !$avatar->profile) { + continue; + } + $url = $avatar->profile->remote_url; + if(!$url || !Helpers::validateUrl($url)) { + continue; + } + try { + $res = Helpers::fetchFromUrl($url); + if( + !is_array($res) || + !isset($res['@context']) || + !isset($res['icon']) || + !isset($res['icon']['type']) || + !isset($res['icon']['url']) || + !Str::endsWith($res['icon']['url'], ['.png', '.jpg', '.jpeg']) + ) { + continue; + } + } catch (\GuzzleHttp\Exception\RequestException $e) { + continue; + } catch(\Illuminate\Http\Client\ConnectionException $e) { + continue; + } + $avatar->remote_url = $res['icon']['url']; + $avatar->save(); + RemoteAvatarFetch::dispatch($avatar->profile); + } + }); + } + + protected function fix() + { + Profile::chunk(5000, function($profiles) { + foreach($profiles as $profile) { + if($profile->domain == null || $profile->private_key) { + continue; + } + $avatar = Avatar::whereProfileId($profile->id)->first(); + if($avatar) { + continue; + } + $avatar = new Avatar; + $avatar->is_remote = true; + $avatar->profile_id = $profile->id; + $avatar->save(); + $this->incr('fixed'); + } + }); + $this->line(' '); + $this->line(' '); + $this->info('Fixed ' . $this->fixed . ' accounts with a blank avatar'); + } + + protected function sync() + { + Avatar::whereIsRemote(true) + ->with('profile') + ->chunk(10, function($avatars) { + foreach($avatars as $avatar) { + RemoteAvatarFetch::dispatch($avatar->profile); + } + }); + } + }