chris/pixelfed
1
0
Fork 0
forked from mirror/pixelfed
Code Pull requests Activity

Merge pull request #4665 from pixelfed/staging

Browse source 
Add Resilient Media Storage
This commit is contained in:
daniel 2023-09-25 01:14:11 -06:00 committed by GitHub
commit 8a89570b4a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 122 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -2,6 +2,9 @@
## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev)
### Added
- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6))
### Federation
- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
- Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d))

11
app/Services/MediaStorageService.php
View file

@ -86,12 +86,11 @@ class MediaStorageService {
$thumbname = array_pop($pt);
$storagePath = implode('/', $p);
$disk = Storage::disk(config('filesystems.cloud'));
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
$url = $disk->url($file);
$thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public');
$thumbUrl = $disk->url($thumbFile);
$media->thumbnail_url = $thumbUrl;
$url = ResilientMediaStorageService::store($storagePath, $path, $name);
if($thumb) {
$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
$media->thumbnail_url = $thumbUrl;
}
$media->cdn_url = $url;
$media->optimized_url = $url;
$media->replicated_at = now();

66
app/Services/ResilientMediaStorageService.php Normal file
View file

@ -0,0 +1,66 @@
<?php
namespace App\Services;
use Storage;
use Illuminate\Http\File;
use Exception;
use GuzzleHttp\Exception\ClientException;
use Aws\S3\Exception\S3Exception;
use GuzzleHttp\Exception\ConnectException;
use League\Flysystem\UnableToWriteFile;
class ResilientMediaStorageService
{
static $attempts = 0;
public static function store($storagePath, $path, $name)
{
return (bool) config_cache('pixelfed.cloud_storage') && (bool) config('media.storage.remote.resilient_mode') ?
self::handleResilientStore($storagePath, $path, $name) :
self::handleStore($storagePath, $path, $name);
}
public static function handleStore($storagePath, $path, $name)
{
return retry(3, function() use($storagePath, $path, $name) {
$baseDisk = (bool) config_cache('pixelfed.cloud_storage') ? config('filesystems.cloud') : 'local';
$disk = Storage::disk($baseDisk);
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
return $disk->url($file);
}, random_int(100, 500));
}
public static function handleResilientStore($storagePath, $path, $name)
{
$attempts = 0;
return retry(4, function() use($storagePath, $path, $name, $attempts) {
self::$attempts++;
usleep(100000);
$baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud');
try {
$disk = Storage::disk($baseDisk);
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
} catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {}
return $disk->url($file);
}, function (int $attempt, Exception $exception) {
return $attempt * 200;
});
}
public static function getAltDriver()
{
$drivers = [];
if(config('filesystems.disks.alt-primary.enabled')) {
$drivers[] = 'alt-primary';
}
if(config('filesystems.disks.alt-secondary.enabled')) {
$drivers[] = 'alt-secondary';
}
if(empty($drivers)) {
return false;
}
$key = array_rand($drivers, 1);
return $drivers[$key];
}
}

28
config/filesystems.php
View file

@ -79,6 +79,34 @@ return [
'throw' => true,
],
'alt-primary' => [
'enabled' => env('ALT_PRI_ENABLED', false),
'driver' => 's3',
'key' => env('ALT_PRI_AWS_ACCESS_KEY_ID'),
'secret' => env('ALT_PRI_AWS_SECRET_ACCESS_KEY'),
'region' => env('ALT_PRI_AWS_DEFAULT_REGION'),
'bucket' => env('ALT_PRI_AWS_BUCKET'),
'visibility' => 'public',
'url' => env('ALT_PRI_AWS_URL'),
'endpoint' => env('ALT_PRI_AWS_ENDPOINT'),
'use_path_style_endpoint' => env('ALT_PRI_AWS_USE_PATH_STYLE_ENDPOINT', false),
'throw' => true,
],
'alt-secondary' => [
'enabled' => env('ALT_SEC_ENABLED', false),
'driver' => 's3',
'key' => env('ALT_SEC_AWS_ACCESS_KEY_ID'),
'secret' => env('ALT_SEC_AWS_SECRET_ACCESS_KEY'),
'region' => env('ALT_SEC_AWS_DEFAULT_REGION'),
'bucket' => env('ALT_SEC_AWS_BUCKET'),
'visibility' => 'public',
'url' => env('ALT_SEC_AWS_URL'),
'endpoint' => env('ALT_SEC_AWS_ENDPOINT'),
'use_path_style_endpoint' => env('ALT_SEC_AWS_USE_PATH_STYLE_ENDPOINT', false),
'throw' => true,
],
'spaces' => [
'driver' => 's3',
'key' => env('DO_SPACES_KEY'),

38
config/media.php
View file

@ -1,24 +1,26 @@
<?php
return [
'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
'exif' => [
'database' => env('MEDIA_EXIF_DATABASE', false),
],
'exif' => [
'database' => env('MEDIA_EXIF_DATABASE', false),
],
'storage' => [
'remote' => [
/*
|--------------------------------------------------------------------------
| Store remote media on cloud/S3
|--------------------------------------------------------------------------
|
| Set this to cache remote media on cloud/S3 filesystem drivers.
| Disabled by default.
|
*/
'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false)
],
]
'storage' => [
'remote' => [
/*
|--------------------------------------------------------------------------
| Store remote media on cloud/S3
|--------------------------------------------------------------------------
|
| Set this to cache remote media on cloud/S3 filesystem drivers.
| Disabled by default.
|
*/
'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false),
'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false),
],
]
];