From 82798b5ea3e77867cb06981ac412a3aee781752e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 00:48:30 -0600 Subject: [PATCH 1/5] Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars --- .../AvatarPipeline/AvatarStorageCleanup.php | 67 +++++++++ .../AvatarStorageLargePurge.php | 80 +++++++++++ app/Jobs/AvatarPipeline/RemoteAvatarFetch.php | 2 +- .../RemoteAvatarFetchFromUrl.php | 1 - app/Services/AvatarService.php | 128 ++++++++++++++++-- app/Services/MediaStorageService.php | 9 +- 6 files changed, 270 insertions(+), 17 deletions(-) create mode 100644 app/Jobs/AvatarPipeline/AvatarStorageCleanup.php create mode 100644 app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php diff --git a/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php b/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php new file mode 100644 index 000000000..230797bf6 --- /dev/null +++ b/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php @@ -0,0 +1,67 @@ +avatar->profile_id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct(Avatar $avatar) + { + $this->avatar = $avatar->withoutRelations(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + AvatarService::cleanup($this->avatar, true); + + return; + } +} diff --git a/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php b/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php new file mode 100644 index 000000000..f432e1e56 --- /dev/null +++ b/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php @@ -0,0 +1,80 @@ +avatar->profile_id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct(Avatar $avatar) + { + $this->avatar = $avatar->withoutRelations(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + $avatar = $this->avatar; + + $disk = AvatarService::disk(); + + $files = collect(AvatarService::storage($avatar)); + + $curFile = Str::of($avatar->cdn_url)->explode('/')->last(); + + $files = $files->filter(function($f) use($curFile) { + return !$curFile || !str_ends_with($f, $curFile); + })->each(function($name) use($disk) { + $disk->delete($name); + }); + + return; + } +} diff --git a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php index df972dd38..4e4a1b2ec 100644 --- a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php +++ b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php @@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue $avatar->remote_url = $icon['url']; $avatar->save(); - MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false); + MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true); return 1; } diff --git a/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php b/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php index 259058385..c8c6820e4 100644 --- a/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php +++ b/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php @@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue $avatar->save(); } - MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true); return 1; diff --git a/app/Services/AvatarService.php b/app/Services/AvatarService.php index 1c5e9e0c1..af578fdef 100644 --- a/app/Services/AvatarService.php +++ b/app/Services/AvatarService.php @@ -3,21 +3,125 @@ namespace App\Services; use Cache; +use Storage; +use Illuminate\Support\Str; +use App\Avatar; use App\Profile; +use App\Jobs\AvatarPipeline\AvatarStorageLargePurge; +use League\Flysystem\UnableToCheckDirectoryExistence; +use League\Flysystem\UnableToRetrieveMetadata; class AvatarService { - public static function get($profile_id) - { - $exists = Cache::get('avatar:' . $profile_id); - if($exists) { - return $exists; - } + public static function get($profile_id) + { + $exists = Cache::get('avatar:' . $profile_id); + if($exists) { + return $exists; + } - $profile = Profile::find($profile_id); - if(!$profile) { - return config('app.url') . '/storage/avatars/default.jpg'; - } - return $profile->avatarUrl(); - } + $profile = Profile::find($profile_id); + if(!$profile) { + return config('app.url') . '/storage/avatars/default.jpg'; + } + return $profile->avatarUrl(); + } + + public static function disk() + { + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + return false; + } + + $driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud'); + $disk = Storage::disk($driver); + + return $disk; + } + + public static function storage(Avatar $avatar) + { + $disk = self::disk(); + + if(!$disk) { + return; + } + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/'; + + return $disk->allFiles($base . $avatar->profile_id); + } + + public static function cleanup($avatar, $confirm = false) + { + if(!$avatar || !$confirm) { + return; + } + + if($avatar->cdn_url == null) { + return; + } + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + return; + } + + $disk = self::disk(); + + if(!$disk) { + return; + } + + $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/'; + + try { + $exists = $disk->directoryExists($base . $avatar->profile_id); + } catch ( + UnableToRetrieveMetadata | + UnableToCheckDirectoryExistence | + Exception $e + ) { + return; + } + + if(!$exists) { + return; + } + + $files = collect($disk->allFiles($base . $avatar->profile_id)); + + if(!$files || !$files->count() || $files->count() === 1) { + return; + } + + if($files->count() > 5) { + AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo'); + return; + } + + $curFile = Str::of($avatar->cdn_url)->explode('/')->last(); + + $files = $files->filter(function($f) use($curFile) { + return !$curFile || !str_ends_with($f, $curFile); + })->each(function($name) use($disk) { + $disk->delete($name); + }); + + return; + } } diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index b547ee39c..128001de2 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController; use GuzzleHttp\Exception\RequestException; use App\Jobs\MediaPipeline\MediaDeletePipeline; use Illuminate\Support\Arr; +use App\Jobs\AvatarPipeline\AvatarStorageCleanup; class MediaStorageService { @@ -29,9 +30,9 @@ class MediaStorageService { return; } - public static function avatar($avatar, $local = false) + public static function avatar($avatar, $local = false, $skipRecentCheck = false) { - return (new self())->fetchAvatar($avatar, $local); + return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck); } public static function head($url) @@ -182,6 +183,7 @@ class MediaStorageService { protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false) { + $queue = random_int(1, 15) > 5 ? 'mmo' : 'low'; $url = $avatar->remote_url; $driver = $local ? 'local' : config('filesystems.cloud'); @@ -205,7 +207,7 @@ class MediaStorageService { $max_size = (int) config('pixelfed.max_avatar_size') * 1000; if(!$skipRecentCheck) { - if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) { + if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) { return; } } @@ -261,6 +263,7 @@ class MediaStorageService { Cache::forget('avatar:' . $avatar->profile_id); AccountService::del($avatar->profile_id); + AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15))); unlink($tmpName); } From 95a1eddcb244420122b299af5110d480858bc3be Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 00:48:56 -0600 Subject: [PATCH 2/5] Update changelog --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 969c9cc99..2ad8dbccd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,11 @@ - Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191)) - Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa)) - Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b)) -- Update ApiV1Controller, hydrate reblog interactions. Fixes #4686 ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb)) -- Update AdminReportController, add `profile_id` to group by. Fixes #4685 ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196)) +- Update ApiV1Controller, hydrate reblog interactions. Fixes ([#4686](https://github.com/pixelfed/pixelfed/issues/4686)) ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb)) +- Update AdminReportController, add `profile_id` to group by. Fixes ([#4685](https://github.com/pixelfed/pixelfed/issues/4685)) ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196)) +- Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511)) +- Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3)) +- Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) From 319ced405413bc2a2745b163301929857bf1a7d5 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 01:32:04 -0600 Subject: [PATCH 3/5] Update CreateAvatar job, add processing constraints and set is_remote attribute --- app/Jobs/AvatarPipeline/CreateAvatar.php | 59 +++++++++++++++++++----- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/app/Jobs/AvatarPipeline/CreateAvatar.php b/app/Jobs/AvatarPipeline/CreateAvatar.php index fd5f94cc7..f773d1590 100644 --- a/app/Jobs/AvatarPipeline/CreateAvatar.php +++ b/app/Jobs/AvatarPipeline/CreateAvatar.php @@ -2,19 +2,25 @@ namespace App\Jobs\AvatarPipeline; -use App\Avatar; -use App\Profile; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\Middleware\WithoutOverlapping; +use App\Avatar; +use App\Profile; -class CreateAvatar implements ShouldQueue +class CreateAvatar implements ShouldQueue, ShouldBeUniqueUntilProcessing { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $profile; + public $profile; + public $tries = 3; + public $maxExceptions = 3; + public $timeout = 900; + public $failOnTimeout = true; /** * Delete the job if its models no longer exist. @@ -22,6 +28,31 @@ class CreateAvatar implements ShouldQueue * @var bool */ public $deleteWhenMissingModels = true; + + /** + * The number of seconds after which the job's unique lock will be released. + * + * @var int + */ + public $uniqueFor = 3600; + + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return 'avatar:create:' . $this->profile->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-create:{$this->profile->id}"))->shared()->dontRelease()]; + } /** * Create a new job instance. @@ -30,7 +61,7 @@ class CreateAvatar implements ShouldQueue */ public function __construct(Profile $profile) { - $this->profile = $profile; + $this->profile = $profile->withoutRelations(); } /** @@ -41,12 +72,18 @@ class CreateAvatar implements ShouldQueue public function handle() { $profile = $this->profile; + $isRemote = (bool) $profile->private_key == null; $path = 'public/avatars/default.jpg'; - $avatar = new Avatar(); - $avatar->profile_id = $profile->id; - $avatar->media_path = $path; - $avatar->change_count = 0; - $avatar->last_processed_at = \Carbon\Carbon::now(); - $avatar->save(); + Avatar::updateOrCreate( + [ + 'profile_id' => $profile->id, + ], + [ + 'media_path' => $path, + 'change_count' => 0, + 'is_remote' => $isRemote, + 'last_processed_at' => now() + ] + ); } } From c37b7cde30189200a28e2ae6a947f1345ce30ab7 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 03:38:31 -0600 Subject: [PATCH 4/5] Add `avatar:storage-deep-clean` command to dispatch avatar storage cleanup jobs --- .../Commands/AvatarStorageDeepClean.php | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 app/Console/Commands/AvatarStorageDeepClean.php diff --git a/app/Console/Commands/AvatarStorageDeepClean.php b/app/Console/Commands/AvatarStorageDeepClean.php new file mode 100644 index 000000000..5840142f5 --- /dev/null +++ b/app/Console/Commands/AvatarStorageDeepClean.php @@ -0,0 +1,115 @@ +info(' ____ _ ______ __ '); + $this->info(' / __ \(_) _____ / / __/__ ____/ / '); + $this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / '); + $this->info(' / ____/ /> info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ '); + $this->info(' '); + $this->info(' Pixelfed Avatar Deep Cleaner'); + $this->line(' '); + $this->info(' Purge/delete old and outdated avatars from remote accounts'); + $this->line(' '); + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + $this->error('Remote avatars are not cached locally, there is nothing to purge. Aborting...'); + exit; + } + + $start = 0; + + if(!$this->confirm('Are you sure you want to proceed?')) { + $this->error('Aborting...'); + exit; + } + + if(!$this->activeCheck()) { + $this->info('Found existing deep cleaning job'); + if(!$this->confirm('Do you want to continue where you left off?')) { + $this->error('Aborting...'); + exit; + } else { + $start = Cache::has('cmd:asdp') ? (int) Cache::get('cmd:asdp') : (int) Storage::get('avatar-deep-clean.json'); + + if($start && $start < 1 || $start > PHP_INT_MAX) { + $this->error('Error fetching cached value'); + $this->error('Aborting...'); + exit; + } + } + } + + $count = Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->count(); + $bar = $this->output->createProgressBar($count); + + foreach(Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->lazyById(10, 'id') as $avatar) { + usleep(random_int(50, 1000)); + $this->counter++; + $this->handleAvatar($avatar); + $bar->advance(); + } + $bar->finish(); + } + + protected function updateCache($id) + { + Cache::put('cmd:asdp', $id); + if($this->counter % 5 === 0) { + Storage::put('avatar-deep-clean.json', $id); + } + } + + protected function activeCheck() + { + if(Storage::exists('avatar-deep-clean.json') || Cache::has('cmd:asdp')) { + return false; + } + + return true; + } + + protected function handleAvatar($avatar) + { + $this->updateCache($avatar->id); + $queues = ['feed', 'mmo', 'feed', 'mmo', 'feed', 'feed', 'mmo', 'low']; + $queue = $queues[random_int(0, 7)]; + AvatarStorageCleanup::dispatch($avatar)->onQueue($queue); + } +} From 4e35f0d32ea7bee4f1a30db6d8d1a73ea88cea8f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Wed, 11 Oct 2023 03:39:35 -0600 Subject: [PATCH 5/5] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ad8dbccd..e1e446bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Added - Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6)) - Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7)) +- Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde)) ### Federation - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e)) @@ -30,6 +31,7 @@ - Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511)) - Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3)) - Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e)) +- Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)