From fb1deb6e28e61738acdc35fcc928ee7222ae741d Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 00:59:24 -0600 Subject: [PATCH 1/2] Add Resilient Media Storage --- app/Services/MediaStorageService.php | 11 ++-- app/Services/ResilientMediaStorageService.php | 66 +++++++++++++++++++ config/filesystems.php | 28 ++++++++ config/media.php | 38 ++++++----- 4 files changed, 119 insertions(+), 24 deletions(-) create mode 100644 app/Services/ResilientMediaStorageService.php diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index b52662d1..b547ee39 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -86,12 +86,11 @@ class MediaStorageService { $thumbname = array_pop($pt); $storagePath = implode('/', $p); - $disk = Storage::disk(config('filesystems.cloud')); - $file = $disk->putFileAs($storagePath, new File($path), $name, 'public'); - $url = $disk->url($file); - $thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public'); - $thumbUrl = $disk->url($thumbFile); - $media->thumbnail_url = $thumbUrl; + $url = ResilientMediaStorageService::store($storagePath, $path, $name); + if($thumb) { + $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname); + $media->thumbnail_url = $thumbUrl; + } $media->cdn_url = $url; $media->optimized_url = $url; $media->replicated_at = now(); diff --git a/app/Services/ResilientMediaStorageService.php b/app/Services/ResilientMediaStorageService.php new file mode 100644 index 00000000..ac1b089a --- /dev/null +++ b/app/Services/ResilientMediaStorageService.php @@ -0,0 +1,66 @@ +putFileAs($storagePath, new File($path), $name, 'public'); + return $disk->url($file); + }, random_int(100, 500)); + } + + public static function handleResilientStore($storagePath, $path, $name) + { + $attempts = 0; + return retry(4, function() use($storagePath, $path, $name, $attempts) { + self::$attempts++; + usleep(100000); + $baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud'); + try { + $disk = Storage::disk($baseDisk); + $file = $disk->putFileAs($storagePath, new File($path), $name, 'public'); + } catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {} + return $disk->url($file); + }, function (int $attempt, Exception $exception) { + return $attempt * 200; + }); + } + + public static function getAltDriver() + { + $drivers = []; + if(config('filesystems.disks.alt-primary.enabled')) { + $drivers[] = 'alt-primary'; + } + if(config('filesystems.disks.alt-secondary.enabled')) { + $drivers[] = 'alt-secondary'; + } + if(empty($drivers)) { + return false; + } + $key = array_rand($drivers, 1); + return $drivers[$key]; + } +} diff --git a/config/filesystems.php b/config/filesystems.php index 6817d5e3..80e63ed9 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -79,6 +79,34 @@ return [ 'throw' => true, ], + 'alt-primary' => [ + 'enabled' => env('ALT_PRI_ENABLED', false), + 'driver' => 's3', + 'key' => env('ALT_PRI_AWS_ACCESS_KEY_ID'), + 'secret' => env('ALT_PRI_AWS_SECRET_ACCESS_KEY'), + 'region' => env('ALT_PRI_AWS_DEFAULT_REGION'), + 'bucket' => env('ALT_PRI_AWS_BUCKET'), + 'visibility' => 'public', + 'url' => env('ALT_PRI_AWS_URL'), + 'endpoint' => env('ALT_PRI_AWS_ENDPOINT'), + 'use_path_style_endpoint' => env('ALT_PRI_AWS_USE_PATH_STYLE_ENDPOINT', false), + 'throw' => true, + ], + + 'alt-secondary' => [ + 'enabled' => env('ALT_SEC_ENABLED', false), + 'driver' => 's3', + 'key' => env('ALT_SEC_AWS_ACCESS_KEY_ID'), + 'secret' => env('ALT_SEC_AWS_SECRET_ACCESS_KEY'), + 'region' => env('ALT_SEC_AWS_DEFAULT_REGION'), + 'bucket' => env('ALT_SEC_AWS_BUCKET'), + 'visibility' => 'public', + 'url' => env('ALT_SEC_AWS_URL'), + 'endpoint' => env('ALT_SEC_AWS_ENDPOINT'), + 'use_path_style_endpoint' => env('ALT_SEC_AWS_USE_PATH_STYLE_ENDPOINT', false), + 'throw' => true, + ], + 'spaces' => [ 'driver' => 's3', 'key' => env('DO_SPACES_KEY'), diff --git a/config/media.php b/config/media.php index b7d6e95c..f550ff29 100644 --- a/config/media.php +++ b/config/media.php @@ -1,24 +1,26 @@ env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true), + 'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true), - 'exif' => [ - 'database' => env('MEDIA_EXIF_DATABASE', false), - ], + 'exif' => [ + 'database' => env('MEDIA_EXIF_DATABASE', false), + ], - 'storage' => [ - 'remote' => [ - /* - |-------------------------------------------------------------------------- - | Store remote media on cloud/S3 - |-------------------------------------------------------------------------- - | - | Set this to cache remote media on cloud/S3 filesystem drivers. - | Disabled by default. - | - */ - 'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false) - ], - ] + 'storage' => [ + 'remote' => [ + /* + |-------------------------------------------------------------------------- + | Store remote media on cloud/S3 + |-------------------------------------------------------------------------- + | + | Set this to cache remote media on cloud/S3 filesystem drivers. + | Disabled by default. + | + */ + 'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false), + + 'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false), + ], + ] ]; From 439638f7d78dc3907cf66bff6aacbf36b1edc250 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Mon, 25 Sep 2023 01:13:14 -0600 Subject: [PATCH 2/2] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b0d55a0..d294f472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev) +### Added +- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6)) + ### Federation - Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e)) - Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d))