Merge pull request #4694 from pixelfed/staging

Update AvatarPipeline, improve refresh logic and garbage collection

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Added
 - Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6))
 - Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7))
+- Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde))
 
 ### Federation
 - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
@@ -25,8 +26,12 @@
 - Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191))
 - Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa))
 - Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b))
-- Update ApiV1Controller, hydrate reblog interactions. Fixes #4686 ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb))
+- Update ApiV1Controller, hydrate reblog interactions. Fixes ([#4686](https://github.com/pixelfed/pixelfed/issues/4686)) ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb))
-- Update AdminReportController, add `profile_id` to group by. Fixes #4685 ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196))
+- Update AdminReportController, add `profile_id` to group by. Fixes ([#4685](https://github.com/pixelfed/pixelfed/issues/4685)) ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196))
+- Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511))
+- Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3))
+- Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e))
+- Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)

app/Console/Commands/AvatarStorageDeepClean.php

@@ -0,0 +1,115 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Cache;
+use Storage;
+use App\Avatar;
+use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
+
+class AvatarStorageDeepClean extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'avatar:storage-deep-clean';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Cleanup avatar storage';
+
+    protected $shouldKeepRunning = true;
+    protected $counter = 0;
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        $this->info('       ____  _           ______         __  ');
+        $this->info('      / __ \(_)  _____  / / __/__  ____/ /  ');
+        $this->info('     / /_/ / / |/_/ _ \/ / /_/ _ \/ __  /   ');
+        $this->info('    / ____/ />  </  __/ / __/  __/ /_/ /    ');
+        $this->info('   /_/   /_/_/|_|\___/_/_/  \___/\__,_/     ');
+        $this->info(' ');
+        $this->info('    Pixelfed Avatar Deep Cleaner');
+        $this->line(' ');
+        $this->info('    Purge/delete old and outdated avatars from remote accounts');
+        $this->line(' ');
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            $this->error('Remote avatars are not cached locally, there is nothing to purge. Aborting...');
+            exit;
+        }
+
+        $start = 0;
+
+        if(!$this->confirm('Are you sure you want to proceed?')) {
+            $this->error('Aborting...');
+            exit;
+        }
+
+        if(!$this->activeCheck()) {
+            $this->info('Found existing deep cleaning job');
+            if(!$this->confirm('Do you want to continue where you left off?')) {
+                $this->error('Aborting...');
+                exit;
+            } else {
+                $start = Cache::has('cmd:asdp') ? (int) Cache::get('cmd:asdp') : (int) Storage::get('avatar-deep-clean.json');
+
+                if($start && $start < 1 || $start > PHP_INT_MAX) {
+                    $this->error('Error fetching cached value');
+                    $this->error('Aborting...');
+                    exit;
+                }
+            }
+        }
+
+        $count = Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->count();
+        $bar = $this->output->createProgressBar($count);
+
+        foreach(Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->lazyById(10, 'id') as $avatar) {
+            usleep(random_int(50, 1000));
+            $this->counter++;
+            $this->handleAvatar($avatar);
+            $bar->advance();
+        }
+        $bar->finish();
+    }
+
+    protected function updateCache($id)
+    {
+        Cache::put('cmd:asdp', $id);
+        if($this->counter % 5 === 0) {
+            Storage::put('avatar-deep-clean.json', $id);
+        }
+    }
+
+    protected function activeCheck()
+    {
+        if(Storage::exists('avatar-deep-clean.json') || Cache::has('cmd:asdp')) {
+            return false;
+        }
+
+        return true;
+    }
+
+    protected function handleAvatar($avatar)
+    {
+        $this->updateCache($avatar->id);
+        $queues = ['feed', 'mmo', 'feed', 'mmo', 'feed', 'feed', 'mmo', 'low'];
+        $queue = $queues[random_int(0, 7)];
+        AvatarStorageCleanup::dispatch($avatar)->onQueue($queue);
+    }
+}

app/Jobs/AvatarPipeline/AvatarStorageCleanup.php

@@ -0,0 +1,67 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+
+class AvatarStorageCleanup implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:cleanup:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        AvatarService::cleanup($this->avatar, true);
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace App\Jobs\AvatarPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Services\AvatarService;
+use App\Avatar;
+use Illuminate\Support\Str;
+
+class AvatarStorageLargePurge implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $avatar;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:storage:lg-purge:' . $this->avatar->profile_id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct(Avatar $avatar)
+    {
+        $this->avatar = $avatar->withoutRelations();
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $avatar = $this->avatar;
+
+        $disk = AvatarService::disk();
+
+        $files = collect(AvatarService::storage($avatar));
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
+}

app/Jobs/AvatarPipeline/CreateAvatar.php

@@ -2,19 +2,25 @@
 
 namespace App\Jobs\AvatarPipeline;
 
-use App\Avatar;
-use App\Profile;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use App\Avatar;
+use App\Profile;
 
-class CreateAvatar implements ShouldQueue
+class CreateAvatar implements ShouldQueue, ShouldBeUniqueUntilProcessing
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
-    protected $profile;
+    public $profile;
+    public $tries = 3;
+    public $maxExceptions = 3;
+    public $timeout = 900;
+    public $failOnTimeout = true;
 
     /**
      * Delete the job if its models no longer exist.
@@ -22,6 +28,31 @@ class CreateAvatar implements ShouldQueue
      * @var bool
      */
     public $deleteWhenMissingModels = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'avatar:create:' . $this->profile->id;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("avatar-create:{$this->profile->id}"))->shared()->dontRelease()];
+    }
     
     /**
      * Create a new job instance.
@@ -30,7 +61,7 @@ class CreateAvatar implements ShouldQueue
      */
     public function __construct(Profile $profile)
     {
-        $this->profile = $profile;
+        $this->profile = $profile->withoutRelations();
     }
 
     /**
@@ -41,12 +72,18 @@ class CreateAvatar implements ShouldQueue
     public function handle()
     {
         $profile = $this->profile;
+        $isRemote = (bool) $profile->private_key == null;
         $path = 'public/avatars/default.jpg';
-        $avatar = new Avatar();
+        Avatar::updateOrCreate(
-        $avatar->profile_id = $profile->id;
+            [
-        $avatar->media_path = $path;
+                'profile_id' => $profile->id,
-        $avatar->change_count = 0;
+            ],
-        $avatar->last_processed_at = \Carbon\Carbon::now();
+            [
-        $avatar->save();
+                'media_path' => $path,
+                'change_count' => 0,
+                'is_remote' => $isRemote,
+                'last_processed_at' => now()
+            ]
+        );
     }
 }

app/Jobs/AvatarPipeline/RemoteAvatarFetch.php

@@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue
 		$avatar->remote_url = $icon['url'];
 		$avatar->save();
 
-		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false);
+		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;
 	}

app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php

@@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue
 			$avatar->save();
 		}
 
-
 		MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
 
 		return 1;

app/Services/AvatarService.php

@@ -3,21 +3,125 @@
 namespace App\Services;
 
 use Cache;
+use Storage;
+use Illuminate\Support\Str;
+use App\Avatar;
 use App\Profile;
+use App\Jobs\AvatarPipeline\AvatarStorageLargePurge;
+use League\Flysystem\UnableToCheckDirectoryExistence;
+use League\Flysystem\UnableToRetrieveMetadata;
 
 class AvatarService
 {
-	public static function get($profile_id)
+    public static function get($profile_id)
-	{
+    {
-		$exists = Cache::get('avatar:' . $profile_id);
+        $exists = Cache::get('avatar:' . $profile_id);
-		if($exists) {
+        if($exists) {
-			return $exists;
+            return $exists;
-		}
+        }
 
-		$profile = Profile::find($profile_id);
+        $profile = Profile::find($profile_id);
-		if(!$profile) {
+        if(!$profile) {
-			return config('app.url') . '/storage/avatars/default.jpg';
+            return config('app.url') . '/storage/avatars/default.jpg';
-		}
+        }
-		return $profile->avatarUrl();
+        return $profile->avatarUrl();
-	}
+    }
+
+    public static function disk()
+    {
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return false;
+        }
+
+        $driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud');
+        $disk = Storage::disk($driver);
+
+        return $disk;
+    }
+
+    public static function storage(Avatar $avatar)
+    {
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        return $disk->allFiles($base . $avatar->profile_id);
+    }
+
+    public static function cleanup($avatar, $confirm = false)
+    {
+        if(!$avatar || !$confirm) {
+            return;
+        }
+
+        if($avatar->cdn_url == null) {
+            return;
+        }
+
+        $storage = [
+            'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
+            'local' => boolval(config_cache('federation.avatars.store_local'))
+        ];
+
+        if(!$storage['cloud'] && !$storage['local']) {
+            return;
+        }
+
+        $disk = self::disk();
+
+        if(!$disk) {
+            return;
+        }
+
+        $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
+
+        try {
+            $exists = $disk->directoryExists($base . $avatar->profile_id);
+        } catch (
+            UnableToRetrieveMetadata |
+            UnableToCheckDirectoryExistence |
+            Exception $e
+        ) {
+            return;
+        }
+
+        if(!$exists) {
+            return;
+        }
+
+        $files = collect($disk->allFiles($base . $avatar->profile_id));
+
+        if(!$files || !$files->count() || $files->count() === 1) {
+            return;
+        }
+
+        if($files->count() > 5) {
+            AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo');
+            return;
+        }
+
+        $curFile = Str::of($avatar->cdn_url)->explode('/')->last();
+
+        $files = $files->filter(function($f) use($curFile) {
+            return !$curFile || !str_ends_with($f, $curFile);
+        })->each(function($name) use($disk) {
+            $disk->delete($name);
+        });
+
+        return;
+    }
 }

app/Services/MediaStorageService.php

@@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController;
 use GuzzleHttp\Exception\RequestException;
 use App\Jobs\MediaPipeline\MediaDeletePipeline;
 use Illuminate\Support\Arr;
+use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
 
 class MediaStorageService {
 
@@ -29,9 +30,9 @@ class MediaStorageService {
 		return;
 	}
 
-	public static function avatar($avatar, $local = false)
+	public static function avatar($avatar, $local = false, $skipRecentCheck = false)
 	{
-		return (new self())->fetchAvatar($avatar, $local);
+		return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
 	}
 
 	public static function head($url)
@@ -182,6 +183,7 @@ class MediaStorageService {
 
 	protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
 	{
+		$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
 		$url = $avatar->remote_url;
 		$driver = $local ? 'local' : config('filesystems.cloud');
 
@@ -205,7 +207,7 @@ class MediaStorageService {
 		$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
 
 		if(!$skipRecentCheck) {
-			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) {
+			if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
 				return;
 			}
 		}
@@ -261,6 +263,7 @@ class MediaStorageService {
 
 		Cache::forget('avatar:' . $avatar->profile_id);
 		AccountService::del($avatar->profile_id);
+		AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
 
 		unlink($tmpName);
 	}