From 28bf8649b371a91ef37bdddeecf3b5510d1b14d1 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 4 Apr 2023 02:35:37 -0600 Subject: [PATCH 1/4] Update Profile model, fix avatar url path generation. Fixes #4041, Fixes #4031, Fixes #3523 --- app/Profile.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/Profile.php b/app/Profile.php index 450520880..0baa33bf6 100644 --- a/app/Profile.php +++ b/app/Profile.php @@ -178,13 +178,21 @@ class Profile extends Model return url('/storage/avatars/default.jpg'); } + if($path === 'public/avatars/default.jpg') { + return url('/storage/avatars/default.jpg'); + } + if(substr($path, 0, 6) !== 'public') { return url('/storage/avatars/default.jpg'); } + if(config('filesystems.default') !== 'local') { + return Storage::url($path); + } + $path = "{$path}?v={$avatar->change_count}"; - return config('app.url') . Storage::url($path); + return url(Storage::url($path)); }); return $url; From 672cccd4337278cfb07570a1be0fd4b3b4b15e43 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 4 Apr 2023 03:46:29 -0600 Subject: [PATCH 2/4] Add media:fix-nonlocal-driver command. Fixes s3 media created with invalid FILESYSTEM_DRIVER=s3 configuration --- app/Console/Commands/FixMediaDriver.php | 133 ++++++++++++++++++ ...MediaFixLocalFilesystemCleanupPipeline.php | 75 ++++++++++ 2 files changed, 208 insertions(+) create mode 100644 app/Console/Commands/FixMediaDriver.php create mode 100644 app/Jobs/MediaPipeline/MediaFixLocalFilesystemCleanupPipeline.php diff --git a/app/Console/Commands/FixMediaDriver.php b/app/Console/Commands/FixMediaDriver.php new file mode 100644 index 000000000..4fe1c95e9 --- /dev/null +++ b/app/Console/Commands/FixMediaDriver.php @@ -0,0 +1,133 @@ +error('Invalid default filesystem, set FILESYSTEM_DRIVER=local to proceed'); + return Command::SUCCESS; + } + + if(config_cache('pixelfed.cloud_storage') == false) { + $this->error('Cloud storage not enabled, exiting...'); + return Command::SUCCESS; + } + + $this->info(' ____ _ ______ __ '); + $this->info(' / __ \(_) _____ / / __/__ ____/ / '); + $this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / '); + $this->info(' / ____/ /> info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ '); + $this->info(' '); + $this->info(' Media Filesystem Fix'); + $this->info(' ====================='); + $this->info(' Fix media that was created when FILESYSTEM_DRIVER=local'); + $this->info(' was not properly set. This command will fix media urls'); + $this->info(' and optionally optimize/generate thumbnails when applicable,'); + $this->info(' clean up temporary local media files and clear the app cache'); + $this->info(' to fix media paths/urls.'); + $this->info(' '); + $this->error(' Remember, FILESYSTEM_DRIVER=local must remain set or you will break things!'); + + if(!$this->confirm('Are you sure you want to perform this command?')) { + $this->info('Exiting...'); + return Command::SUCCESS; + } + + $optimize = $this->choice( + 'Do you want to optimize media and generate thumbnails? This will store s3 locally and re-upload optimized versions.', + ['no', 'yes'], + 1 + ); + + $cloud = Storage::disk(config('filesystems.cloud')); + $mountManager = new MountManager([ + 's3' => $cloud->getDriver(), + 'local' => Storage::disk('local')->getDriver(), + ]); + + $this->info('Fixing media, this may take a while...'); + $this->line(' '); + $bar = $this->output->createProgressBar(Media::whereNotNull('status_id')->whereNull('cdn_url')->count()); + $bar->start(); + + foreach(Media::whereNotNull('status_id')->whereNull('cdn_url')->lazyById(20) as $media) { + if($cloud->exists($media->media_path)) { + if($optimize === 'yes') { + $mountManager->copy( + 's3://' . $media->media_path, + 'local://' . $media->media_path + ); + sleep(1); + if(empty($media->original_sha256)) { + $hash = \hash_file('sha256', Storage::disk('local')->path($media->media_path)); + $media->original_sha256 = $hash; + $media->save(); + sleep(1); + } + if( + $media->mime && + in_array($media->mime, [ + 'image/jpeg', + 'image/png', + 'image/webp' + ]) + ) { + ImageOptimize::dispatchSync($media); + sleep(3); + } + } else { + $media->cdn_url = $cloud->url($media->media_path); + $media->save(); + } + } + $bar->advance(); + } + + $bar->finish(); + $this->line(' '); + $this->line(' '); + + $this->callSilently('cache:clear'); + + $this->info('Successfully fixed media paths and cleared cached!'); + + if($optimize === 'yes') { + MediaFixLocalFilesystemCleanupPipeline::dispatch()->delay(now()->addMinutes(15))->onQueue('default'); + $this->line(' '); + $this->info('A cleanup job has been dispatched to delete media stored locally, it may take a few minutes to process!'); + } + + $this->line(' '); + return Command::SUCCESS; + } +} diff --git a/app/Jobs/MediaPipeline/MediaFixLocalFilesystemCleanupPipeline.php b/app/Jobs/MediaPipeline/MediaFixLocalFilesystemCleanupPipeline.php new file mode 100644 index 000000000..bbd3851b9 --- /dev/null +++ b/app/Jobs/MediaPipeline/MediaFixLocalFilesystemCleanupPipeline.php @@ -0,0 +1,75 @@ +chunk(20, function ($medias) use($disk, $cloud) { + foreach($medias as $media) { + if(!str_starts_with($media->media_path, 'public')) { + continue; + } + + if($disk->exists($media->media_path) && $cloud->exists($media->media_path)) { + $disk->delete($media->media_path); + } + + if($media->thumbnail_path) { + if($disk->exists($media->thumbnail_path)) { + $disk->delete($media->thumbnail_path); + } + } + + $paths = explode('/', $media->media_path); + if(count($paths) === 7) { + array_pop($paths); + $baseDir = implode('/', $paths); + + if(count($disk->allFiles($baseDir)) === 0) { + $disk->deleteDirectory($baseDir); + + array_pop($paths); + $baseDir = implode('/', $paths); + + if(count($disk->allFiles($baseDir)) === 0) { + $disk->deleteDirectory($baseDir); + + array_pop($paths); + $baseDir = implode('/', $paths); + + if(count($disk->allFiles($baseDir)) === 0) { + $disk->deleteDirectory($baseDir); + } + } + } + } + } + }); + } +} From 573c88d7b505d0e482a71bd902654c8bf080f97c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 4 Apr 2023 03:52:32 -0600 Subject: [PATCH 3/4] Update filesystem config, change FILESYSTEM_DRIVER env variable to DANGEROUSLY_SET_FILESYSTEM_DRIVER and remove from default env configs. Changing the default filesystem should be avoided, use FILESYSTEM_CLOUD for s3 support, otherwise you can break things --- .env.docker | 1 - .env.example | 1 - config/filesystems.php | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.env.docker b/.env.docker index 8ff20d334..ce4cfe87c 100644 --- a/.env.docker +++ b/.env.docker @@ -101,7 +101,6 @@ NODEINFO=true WEBFINGER=true ## S3 -FILESYSTEM_DRIVER=local FILESYSTEM_CLOUD=s3 PF_ENABLE_CLOUD=false #AWS_ACCESS_KEY_ID= diff --git a/.env.example b/.env.example index 1e453aed0..d4d7228d1 100644 --- a/.env.example +++ b/.env.example @@ -68,7 +68,6 @@ MAIL_FROM_NAME="Pixelfed" ## S3 Configuration (Post-Installer) PF_ENABLE_CLOUD=false -FILESYSTEM_DRIVER=local FILESYSTEM_CLOUD=s3 #AWS_ACCESS_KEY_ID= #AWS_SECRET_ACCESS_KEY= diff --git a/config/filesystems.php b/config/filesystems.php index 0d2a27743..6817d5e34 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -13,7 +13,7 @@ return [ | */ - 'default' => env('FILESYSTEM_DRIVER', 'local'), + 'default' => env('DANGEROUSLY_SET_FILESYSTEM_DRIVER', 'local'), /* |-------------------------------------------------------------------------- From 014a4d9c51ca0627e3e2a78891125af16869b655 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 4 Apr 2023 03:54:08 -0600 Subject: [PATCH 4/4] Update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6deca5254..7ece6c695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,19 @@ ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.5...dev) +### Added +- New media:fix-nonlocal-driver command. Fixes s3 media created with invalid FILESYSTEM_DRIVER=s3 configuration ([672cccd4](https://github.com/pixelfed/pixelfed/commit/672cccd4)) + ### Updates - Update ApiV1Controller, fix blocking remote accounts. Closes #4256 ([8e71e0c0](https://github.com/pixelfed/pixelfed/commit/8e71e0c0)) - Update ComposeController, fix postgres location search. Closes #4242 and #4239 ([64a4a006](https://github.com/pixelfed/pixelfed/commit/64a4a006)) - Update app.js, add title attribute to iframe embeds to comply with accessibility requirements ([4d72b9e3](https://github.com/pixelfed/pixelfed/commit/4d72b9e3)) - Update MediaPathService, fix story path ([aebbad96](https://github.com/pixelfed/pixelfed/commit/aebbad96)) - Update Story v1.1 api endpoints ([855e9626](https://github.com/pixelfed/pixelfed/commit/855e9626)) +- Update ApiV1Controller, filter mute/blocks on statuses/context and statuses/replies endpoints ([73aa01e8](https://github.com/pixelfed/pixelfed/commit/73aa01e8)) +- Update filesystems, store all files as public by default and add default permissions. Fixes #4273, #4275. Closes #3825 ([22da2647](https://github.com/pixelfed/pixelfed/commit/22da2647)) +- Update Profile model, fix avatar url path generation. Fixes #4041, Fixes #4031, Fixes #3523 ([28bf8649](https://github.com/pixelfed/pixelfed/commit/28bf8649)) +- Update filesystem config, change FILESYSTEM_DRIVER env variable to DANGEROUSLY_SET_FILESYSTEM_DRIVER and remove from default env configs. Changing the default filesystem should be avoided, use FILESYSTEM_CLOUD for s3 support, otherwise you can break things ([573c88d7](https://github.com/pixelfed/pixelfed/commit/573c88d7)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.5 (2023-03-25)](https://github.com/pixelfed/pixelfed/compare/v0.11.4...v0.11.5)