mirror
/
pixelfed
mirror of https://github.com/pixelfed/pixelfed.git
1
0
Fork 1
Code Issues Releases Wiki Activity

Merge pull request #4694 from pixelfed/staging 

Update AvatarPipeline, improve refresh logic and garbage collection
This commit is contained in:
daniel 2023-10-11 03:42:21 -06:00 committed by GitHub
parent ffa44c4fad 4e35f0d32e
commit 8f4f64d737
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 440 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CHANGELOG.md
View File

@ -5,6 +5,7 @@
### Added
- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6))
- Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7))
- Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde))
### Federation
- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
@ -25,8 +26,12 @@
- Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191))
- Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa))
- Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b))
- Update ApiV1Controller, hydrate reblog interactions. Fixes #4686 ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb))
- Update AdminReportController, add `profile_id` to group by. Fixes #4685 ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196))
- Update ApiV1Controller, hydrate reblog interactions. Fixes ([#4686](https://github.com/pixelfed/pixelfed/issues/4686)) ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb))
- Update AdminReportController, add `profile_id` to group by. Fixes ([#4685](https://github.com/pixelfed/pixelfed/issues/4685)) ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196))
- Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511))
- Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3))
- Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e))
- Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40))
- ([](https://github.com/pixelfed/pixelfed/commit/))
## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)

115
app/Console/Commands/AvatarStorageDeepClean.php Normal file
View File

@ -0,0 +1,115 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use Cache;
use Storage;
use App\Avatar;
use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
class AvatarStorageDeepClean extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'avatar:storage-deep-clean';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Cleanup avatar storage';
protected $shouldKeepRunning = true;
protected $counter = 0;
/**
* Execute the console command.
*/
public function handle(): void
{
$this->info(' ____ _ ______ __ ');
$this->info(' / __ \(_) _____ / / __/__ ____/ / ');
$this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / ');
$this->info(' / ____/ /> </ __/ / __/ __/ /_/ / ');
$this->info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ ');
$this->info(' ');
$this->info(' Pixelfed Avatar Deep Cleaner');
$this->line(' ');
$this->info(' Purge/delete old and outdated avatars from remote accounts');
$this->line(' ');
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
if(!$storage['cloud'] && !$storage['local']) {
$this->error('Remote avatars are not cached locally, there is nothing to purge. Aborting...');
exit;
}
$start = 0;
if(!$this->confirm('Are you sure you want to proceed?')) {
$this->error('Aborting...');
exit;
}
if(!$this->activeCheck()) {
$this->info('Found existing deep cleaning job');
if(!$this->confirm('Do you want to continue where you left off?')) {
$this->error('Aborting...');
exit;
} else {
$start = Cache::has('cmd:asdp') ? (int) Cache::get('cmd:asdp') : (int) Storage::get('avatar-deep-clean.json');
if($start && $start < 1 || $start > PHP_INT_MAX) {
$this->error('Error fetching cached value');
$this->error('Aborting...');
exit;
}
}
}
$count = Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->count();
$bar = $this->output->createProgressBar($count);
foreach(Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->lazyById(10, 'id') as $avatar) {
usleep(random_int(50, 1000));
$this->counter++;
$this->handleAvatar($avatar);
$bar->advance();
}
$bar->finish();
}
protected function updateCache($id)
{
Cache::put('cmd:asdp', $id);
if($this->counter % 5 === 0) {
Storage::put('avatar-deep-clean.json', $id);
}
}
protected function activeCheck()
{
if(Storage::exists('avatar-deep-clean.json') || Cache::has('cmd:asdp')) {
return false;
}
return true;
}
protected function handleAvatar($avatar)
{
$this->updateCache($avatar->id);
$queues = ['feed', 'mmo', 'feed', 'mmo', 'feed', 'feed', 'mmo', 'low'];
$queue = $queues[random_int(0, 7)];
AvatarStorageCleanup::dispatch($avatar)->onQueue($queue);
}
}

67
app/Jobs/AvatarPipeline/AvatarStorageCleanup.php Normal file
View File

@ -0,0 +1,67 @@
<?php
namespace App\Jobs\AvatarPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Services\AvatarService;
use App\Avatar;
class AvatarStorageCleanup implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $avatar;
public $tries = 3;
public $maxExceptions = 3;
public $timeout = 900;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'avatar:storage:cleanup:' . $this->avatar->profile_id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct(Avatar $avatar)
{
$this->avatar = $avatar->withoutRelations();
}
/**
* Execute the job.
*/
public function handle(): void
{
AvatarService::cleanup($this->avatar, true);
return;
}
}

80
app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php Normal file
View File

@ -0,0 +1,80 @@
<?php
namespace App\Jobs\AvatarPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Services\AvatarService;
use App\Avatar;
use Illuminate\Support\Str;
class AvatarStorageLargePurge implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $avatar;
public $tries = 3;
public $maxExceptions = 3;
public $timeout = 900;
public $failOnTimeout = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'avatar:storage:lg-purge:' . $this->avatar->profile_id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*/
public function __construct(Avatar $avatar)
{
$this->avatar = $avatar->withoutRelations();
}
/**
* Execute the job.
*/
public function handle(): void
{
$avatar = $this->avatar;
$disk = AvatarService::disk();
$files = collect(AvatarService::storage($avatar));
$curFile = Str::of($avatar->cdn_url)->explode('/')->last();
$files = $files->filter(function($f) use($curFile) {
return !$curFile || !str_ends_with($f, $curFile);
})->each(function($name) use($disk) {
$disk->delete($name);
});
return;
}
}

59
app/Jobs/AvatarPipeline/CreateAvatar.php
View File

@ -2,19 +2,25 @@
namespace App\Jobs\AvatarPipeline;
use App\Avatar;
use App\Profile;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Avatar;
use App\Profile;
class CreateAvatar implements ShouldQueue
class CreateAvatar implements ShouldQueue, ShouldBeUniqueUntilProcessing
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $profile;
public $profile;
public $tries = 3;
public $maxExceptions = 3;
public $timeout = 900;
public $failOnTimeout = true;
/**
* Delete the job if its models no longer exist.
@ -23,6 +29,31 @@ class CreateAvatar implements ShouldQueue
*/
public $deleteWhenMissingModels = true;
/**
* The number of seconds after which the job's unique lock will be released.
*
* @var int
*/
public $uniqueFor = 3600;
/**
* Get the unique ID for the job.
*/
public function uniqueId(): string
{
return 'avatar:create:' . $this->profile->id;
}
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping("avatar-create:{$this->profile->id}"))->shared()->dontRelease()];
}
/**
* Create a new job instance.
*
@ -30,7 +61,7 @@ class CreateAvatar implements ShouldQueue
*/
public function __construct(Profile $profile)
{
$this->profile = $profile;
$this->profile = $profile->withoutRelations();
}
/**
@ -41,12 +72,18 @@ class CreateAvatar implements ShouldQueue
public function handle()
{
$profile = $this->profile;
$isRemote = (bool) $profile->private_key == null;
$path = 'public/avatars/default.jpg';
$avatar = new Avatar();
$avatar->profile_id = $profile->id;
$avatar->media_path = $path;
$avatar->change_count = 0;
$avatar->last_processed_at = \Carbon\Carbon::now();
$avatar->save();
Avatar::updateOrCreate(
[
'profile_id' => $profile->id,
],
[
'media_path' => $path,
'change_count' => 0,
'is_remote' => $isRemote,
'last_processed_at' => now()
]
);
}
}

2
app/Jobs/AvatarPipeline/RemoteAvatarFetch.php
View File

@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue
$avatar->remote_url = $icon['url'];
$avatar->save();
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false);
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
return 1;
}

1
app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php
View File

@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue
$avatar->save();
}
MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true);
return 1;

104
app/Services/AvatarService.php
View File

@ -3,7 +3,13 @@
namespace App\Services;
use Cache;
use Storage;
use Illuminate\Support\Str;
use App\Avatar;
use App\Profile;
use App\Jobs\AvatarPipeline\AvatarStorageLargePurge;
use League\Flysystem\UnableToCheckDirectoryExistence;
use League\Flysystem\UnableToRetrieveMetadata;
class AvatarService
{
@ -20,4 +26,102 @@ class AvatarService
}
return $profile->avatarUrl();
}
public static function disk()
{
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
if(!$storage['cloud'] && !$storage['local']) {
return false;
}
$driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud');
$disk = Storage::disk($driver);
return $disk;
}
public static function storage(Avatar $avatar)
{
$disk = self::disk();
if(!$disk) {
return;
}
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
$base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
return $disk->allFiles($base . $avatar->profile_id);
}
public static function cleanup($avatar, $confirm = false)
{
if(!$avatar || !$confirm) {
return;
}
if($avatar->cdn_url == null) {
return;
}
$storage = [
'cloud' => boolval(config_cache('pixelfed.cloud_storage')),
'local' => boolval(config_cache('federation.avatars.store_local'))
];
if(!$storage['cloud'] && !$storage['local']) {
return;
}
$disk = self::disk();
if(!$disk) {
return;
}
$base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/';
try {
$exists = $disk->directoryExists($base . $avatar->profile_id);
} catch (
UnableToRetrieveMetadata |
UnableToCheckDirectoryExistence |
Exception $e
) {
return;
}
if(!$exists) {
return;
}
$files = collect($disk->allFiles($base . $avatar->profile_id));
if(!$files || !$files->count() || $files->count() === 1) {
return;
}
if($files->count() > 5) {
AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo');
return;
}
$curFile = Str::of($avatar->cdn_url)->explode('/')->last();
$files = $files->filter(function($f) use($curFile) {
return !$curFile || !str_ends_with($f, $curFile);
})->each(function($name) use($disk) {
$disk->delete($name);
});
return;
}
}

9
app/Services/MediaStorageService.php
View File

@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController;
use GuzzleHttp\Exception\RequestException;
use App\Jobs\MediaPipeline\MediaDeletePipeline;
use Illuminate\Support\Arr;
use App\Jobs\AvatarPipeline\AvatarStorageCleanup;
class MediaStorageService {
@ -29,9 +30,9 @@ class MediaStorageService {
return;
}
public static function avatar($avatar, $local = false)
public static function avatar($avatar, $local = false, $skipRecentCheck = false)
{
return (new self())->fetchAvatar($avatar, $local);
return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck);
}
public static function head($url)
@ -182,6 +183,7 @@ class MediaStorageService {
protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false)
{
$queue = random_int(1, 15) > 5 ? 'mmo' : 'low';
$url = $avatar->remote_url;
$driver = $local ? 'local' : config('filesystems.cloud');
@ -205,7 +207,7 @@ class MediaStorageService {
$max_size = (int) config('pixelfed.max_avatar_size') * 1000;
if(!$skipRecentCheck) {
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) {
if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) {
return;
}
}
@ -261,6 +263,7 @@ class MediaStorageService {
Cache::forget('avatar:' . $avatar->profile_id);
AccountService::del($avatar->profile_id);
AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15)));
unlink($tmpName);
}