chris
/
pixelfed
forked from mirror/pixelfed
1
0
Fork 0
Code Pull Requests Activity

Merge pull request #3953 from pixelfed/staging 

Update MediaS3GarbageCollector commmand, disable logging by default and optimize huge invocations
This commit is contained in:
daniel 2022-12-17 21:45:30 -07:00 committed by GitHub
parent 7b4d117eac b3c4d8c47c
commit 15c70e7bbc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 58 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -56,6 +56,7 @@
- Update InboxPipeline, add inbox job queue and separate http sig validation from activity handling ([e6c1604d](https://github.com/pixelfed/pixelfed/commit/e6c1604d))
- Update InboxPipeline, dispatch Follow/Accept Follow jobs to follow queue ([f62d2494](https://github.com/pixelfed/pixelfed/commit/f62d2494))
- Add MediaS3GarbageCollector command to clear local media after uploaded to S3 disks after 12 hours ([b8c3f153](https://github.com/pixelfed/pixelfed/commit/b8c3f153))
- Update MediaS3GarbageCollector command, disable logging by default and optimize huge invocations ([a14af93b](https://github.com/pixelfed/pixelfed/commit/a14af93b))
- ([](https://github.com/pixelfed/pixelfed/commit/))
## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)

59
app/Console/Commands/MediaS3GarbageCollector.php
View File

@ -15,7 +15,7 @@ class MediaS3GarbageCollector extends Command
*
* @var string
*/
protected $signature = 'media:s3gc {--limit=200}';
protected $signature = 'media:s3gc {--limit=200} {--huge} {--log-errors}';
/**
* The console command description.
@ -54,8 +54,23 @@ class MediaS3GarbageCollector extends Command
}
$limit = $this->option('limit');
$hugeMode = $this->option('huge');
$log = $this->option('log-errors');
if($limit > 2000 && !$hugeMode) {
$this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag');
return;
}
$minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first()->id;
return $hugeMode ?
$this->hugeMode($minId, $limit, $log) :
$this->regularMode($minId, $limit, $log);
}
protected function regularMode($minId, $limit, $log)
{
$gc = Media::whereRemoteMedia(false)
->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
->whereNot('version', '4')
@ -84,7 +99,9 @@ class MediaS3GarbageCollector extends Command
$media->save();
}
} else {
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
if($log) {
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
}
}
$bar->advance();
}
@ -96,4 +113,42 @@ class MediaS3GarbageCollector extends Command
}
return 0;
}
protected function hugeMode($minId, $limit, $log)
{
$cloudDisk = Storage::disk(config('filesystems.cloud'));
$localDisk = Storage::disk('local');
$bar = $this->output->createProgressBar($limit);
$bar->start();
Media::whereRemoteMedia(false)
->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
->whereNot('version', '4')
->where('id', '<', $minId)
->chunk(50, function($medias) use($cloudDisk, $localDisk, $bar, $log) {
foreach($medias as $media) {
if($cloudDisk->exists($media->media_path)) {
if( $localDisk->exists($media->media_path)) {
$localDisk->delete($media->media_path);
$media->version = 4;
$media->save();
$totalSize = $totalSize + $media->size;
} else {
$media->version = 4;
$media->save();
}
} else {
if($log) {
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
}
}
$bar->advance();
}
});
$bar->finish();
$this->line(' ');
$this->info('Finished!');
}
}