1
0
Fork 0

Merge pull request #3950 from pixelfed/staging

Staging
This commit is contained in:
daniel 2022-12-17 20:41:25 -07:00 committed by GitHub
commit eec807458c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 166 additions and 23 deletions

View File

@ -55,6 +55,7 @@
- Update ApiV1Controller, remove min avatar size limit, fixes #3715 ([2b0db812](https://github.com/pixelfed/pixelfed/commit/2b0db812))
- Update InboxPipeline, add inbox job queue and separate http sig validation from activity handling ([e6c1604d](https://github.com/pixelfed/pixelfed/commit/e6c1604d))
- Update InboxPipeline, dispatch Follow/Accept Follow jobs to follow queue ([f62d2494](https://github.com/pixelfed/pixelfed/commit/f62d2494))
- Add MediaS3GarbageCollector command to clear local media after uploaded to S3 disks after 12 hours ([b8c3f153](https://github.com/pixelfed/pixelfed/commit/b8c3f153))
- ([](https://github.com/pixelfed/pixelfed/commit/))
## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)

View File

@ -40,22 +40,20 @@ class CatchUnoptimizedMedia extends Command
*/
public function handle()
{
DB::transaction(function() {
Media::whereNull('processed_at')
->where('skip_optimize', '!=', true)
->whereNull('remote_url')
->whereNotNull('status_id')
->whereNotNull('media_path')
->where('created_at', '>', now()->subHours(1))
->whereIn('mime', [
'image/jpeg',
'image/png',
])
->chunk(50, function($medias) {
foreach ($medias as $media) {
ImageOptimize::dispatch($media);
}
});
});
Media::whereNull('processed_at')
->where('created_at', '>', now()->subHours(1))
->where('skip_optimize', '!=', true)
->whereNull('remote_url')
->whereNotNull('status_id')
->whereNotNull('media_path')
->whereIn('mime', [
'image/jpeg',
'image/png',
])
->chunk(50, function($medias) {
foreach ($medias as $media) {
ImageOptimize::dispatch($media);
}
});
}
}

View File

@ -4,7 +4,6 @@ namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\{Media, Status};
use Carbon\Carbon;
use App\Services\MediaStorageService;
class MediaGarbageCollector extends Command
@ -40,11 +39,10 @@ class MediaGarbageCollector extends Command
*/
public function handle()
{
$limit = 20000;
$limit = 500;
$gc = Media::whereNull('status_id')
->where('created_at', '<', Carbon::now()->subHours(12)->toDateTimeString())
->orderBy('created_at','asc')
->where('created_at', '<', now()->subHours(2)->toDateTimeString())
->take($limit)
->get();

View File

@ -0,0 +1,98 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Media;
use App\Status;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
class MediaS3GarbageCollector extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'media:s3gc {--limit=200}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Delete (local) media uploads that exist on S3';
/**
* Create a new command instance.
*
* @return void
*/
public function __construct()
{
parent::__construct();
}
/**
* Execute the console command.
*
* @return int
*/
public function handle()
{
$enabled = config('pixelfed.cloud_storage');
if(!$enabled) {
$this->error('Cloud storage not enabled. Exiting...');
return;
}
$deleteEnabled = config('media.delete_local_after_cloud');
if(!$deleteEnabled) {
$this->error('Delete local storage after cloud upload is not enabled');
return;
}
$limit = $this->option('limit');
$minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first()->id;
$gc = Media::whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
->whereNot('version', '4')
->where('id', '<', $minId)
->inRandomOrder()
->take($limit)
->get();
$totalSize = 0;
$bar = $this->output->createProgressBar($gc->count());
$bar->start();
$cloudDisk = Storage::disk(config('filesystems.cloud'));
$localDisk = Storage::disk('local');
foreach($gc as $media) {
if(
$cloudDisk->exists($media->media_path)
) {
if( $localDisk->exists($media->media_path)) {
$localDisk->delete($media->media_path);
$media->version = 4;
$media->save();
$totalSize = $totalSize + $media->size;
} else {
$media->version = 4;
$media->save();
}
} else {
Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
}
$bar->advance();
}
$bar->finish();
$this->line(' ');
$this->info('Finished!');
if($totalSize) {
$this->info('Cleared ' . $totalSize . ' bytes of media from local disk!');
}
return 0;
}
}

View File

@ -25,13 +25,17 @@ class Kernel extends ConsoleKernel
*/
protected function schedule(Schedule $schedule)
{
$schedule->command('media:optimize')->hourly();
$schedule->command('media:gc')->hourly();
$schedule->command('media:optimize')->hourlyAt(40);
$schedule->command('media:gc')->hourlyAt(5);
$schedule->command('horizon:snapshot')->everyFiveMinutes();
$schedule->command('story:gc')->everyFiveMinutes();
$schedule->command('gc:failedjobs')->dailyAt(3);
$schedule->command('gc:passwordreset')->dailyAt('09:41');
$schedule->command('gc:sessions')->twiceDaily(13, 23);
if(config('pixelfed.cloud_storage') && config('media.delete_local_after_cloud')) {
$schedule->command('media:s3gc')->hourlyAt(15);
}
}
/**

View File

@ -89,6 +89,12 @@ return [
'emergency' => [
'path' => storage_path('logs/laravel.log'),
],
'media' => [
'driver' => 'single',
'path' => storage_path('logs/media.log'),
'level' => 'debug'
],
],
];

View File

@ -1,6 +1,8 @@
<?php
return [
'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
'exif' => [
'database' => env('MEDIA_EXIF_DATABASE', false),
],

View File

@ -0,0 +1,36 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::table('media', function (Blueprint $table) {
$table->index('status_id');
$table->index('replicated_at');
$table->index('version');
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::table('media', function (Blueprint $table) {
$table->dropIndex('media_status_id_index');
$table->dropIndex('media_replicated_at_index');
$table->dropIndex('media_version_index');
});
}
};