forked from mirror/pixelfed
Merge pull request #4665 from pixelfed/staging
Add Resilient Media Storage
This commit is contained in:
daniel
GitHub
commit
8a89570b4a
5 changed files with 122 additions and 24 deletions
|
|
@ -2,6 +2,9 @@
|
||||||
|
|
||||||
## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev)
|
## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev)
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6))
|
||||||
|
|
||||||
### Federation
|
### Federation
|
||||||
- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
|
- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
|
||||||
- Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d))
|
- Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d))
|
||||||
|
|
|
||||||
|
|
@ -86,12 +86,11 @@ class MediaStorageService {
|
||||||
$thumbname = array_pop($pt);
|
$thumbname = array_pop($pt);
|
||||||
$storagePath = implode('/', $p);
|
$storagePath = implode('/', $p);
|
||||||
|
|
||||||
$disk = Storage::disk(config('filesystems.cloud'));
|
$url = ResilientMediaStorageService::store($storagePath, $path, $name);
|
||||||
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
|
if($thumb) {
|
||||||
$url = $disk->url($file);
|
$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
|
||||||
$thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public');
|
$media->thumbnail_url = $thumbUrl;
|
||||||
$thumbUrl = $disk->url($thumbFile);
|
}
|
||||||
$media->thumbnail_url = $thumbUrl;
|
|
||||||
$media->cdn_url = $url;
|
$media->cdn_url = $url;
|
||||||
$media->optimized_url = $url;
|
$media->optimized_url = $url;
|
||||||
$media->replicated_at = now();
|
$media->replicated_at = now();
|
||||||
|
|
|
||||||
66
app/Services/ResilientMediaStorageService.php
Normal file
66
app/Services/ResilientMediaStorageService.php
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Services;
|
||||||
|
|
||||||
|
use Storage;
|
||||||
|
use Illuminate\Http\File;
|
||||||
|
use Exception;
|
||||||
|
use GuzzleHttp\Exception\ClientException;
|
||||||
|
use Aws\S3\Exception\S3Exception;
|
||||||
|
use GuzzleHttp\Exception\ConnectException;
|
||||||
|
use League\Flysystem\UnableToWriteFile;
|
||||||
|
|
||||||
|
class ResilientMediaStorageService
|
||||||
|
{
|
||||||
|
static $attempts = 0;
|
||||||
|
|
||||||
|
public static function store($storagePath, $path, $name)
|
||||||
|
{
|
||||||
|
return (bool) config_cache('pixelfed.cloud_storage') && (bool) config('media.storage.remote.resilient_mode') ?
|
||||||
|
self::handleResilientStore($storagePath, $path, $name) :
|
||||||
|
self::handleStore($storagePath, $path, $name);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static function handleStore($storagePath, $path, $name)
|
||||||
|
{
|
||||||
|
return retry(3, function() use($storagePath, $path, $name) {
|
||||||
|
$baseDisk = (bool) config_cache('pixelfed.cloud_storage') ? config('filesystems.cloud') : 'local';
|
||||||
|
$disk = Storage::disk($baseDisk);
|
||||||
|
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
|
||||||
|
return $disk->url($file);
|
||||||
|
}, random_int(100, 500));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static function handleResilientStore($storagePath, $path, $name)
|
||||||
|
{
|
||||||
|
$attempts = 0;
|
||||||
|
return retry(4, function() use($storagePath, $path, $name, $attempts) {
|
||||||
|
self::$attempts++;
|
||||||
|
usleep(100000);
|
||||||
|
$baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud');
|
||||||
|
try {
|
||||||
|
$disk = Storage::disk($baseDisk);
|
||||||
|
$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
|
||||||
|
} catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {}
|
||||||
|
return $disk->url($file);
|
||||||
|
}, function (int $attempt, Exception $exception) {
|
||||||
|
return $attempt * 200;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public static function getAltDriver()
|
||||||
|
{
|
||||||
|
$drivers = [];
|
||||||
|
if(config('filesystems.disks.alt-primary.enabled')) {
|
||||||
|
$drivers[] = 'alt-primary';
|
||||||
|
}
|
||||||
|
if(config('filesystems.disks.alt-secondary.enabled')) {
|
||||||
|
$drivers[] = 'alt-secondary';
|
||||||
|
}
|
||||||
|
if(empty($drivers)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
$key = array_rand($drivers, 1);
|
||||||
|
return $drivers[$key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -79,6 +79,34 @@ return [
|
||||||
'throw' => true,
|
'throw' => true,
|
||||||
],
|
],
|
||||||
|
|
||||||
|
'alt-primary' => [
|
||||||
|
'enabled' => env('ALT_PRI_ENABLED', false),
|
||||||
|
'driver' => 's3',
|
||||||
|
'key' => env('ALT_PRI_AWS_ACCESS_KEY_ID'),
|
||||||
|
'secret' => env('ALT_PRI_AWS_SECRET_ACCESS_KEY'),
|
||||||
|
'region' => env('ALT_PRI_AWS_DEFAULT_REGION'),
|
||||||
|
'bucket' => env('ALT_PRI_AWS_BUCKET'),
|
||||||
|
'visibility' => 'public',
|
||||||
|
'url' => env('ALT_PRI_AWS_URL'),
|
||||||
|
'endpoint' => env('ALT_PRI_AWS_ENDPOINT'),
|
||||||
|
'use_path_style_endpoint' => env('ALT_PRI_AWS_USE_PATH_STYLE_ENDPOINT', false),
|
||||||
|
'throw' => true,
|
||||||
|
],
|
||||||
|
|
||||||
|
'alt-secondary' => [
|
||||||
|
'enabled' => env('ALT_SEC_ENABLED', false),
|
||||||
|
'driver' => 's3',
|
||||||
|
'key' => env('ALT_SEC_AWS_ACCESS_KEY_ID'),
|
||||||
|
'secret' => env('ALT_SEC_AWS_SECRET_ACCESS_KEY'),
|
||||||
|
'region' => env('ALT_SEC_AWS_DEFAULT_REGION'),
|
||||||
|
'bucket' => env('ALT_SEC_AWS_BUCKET'),
|
||||||
|
'visibility' => 'public',
|
||||||
|
'url' => env('ALT_SEC_AWS_URL'),
|
||||||
|
'endpoint' => env('ALT_SEC_AWS_ENDPOINT'),
|
||||||
|
'use_path_style_endpoint' => env('ALT_SEC_AWS_USE_PATH_STYLE_ENDPOINT', false),
|
||||||
|
'throw' => true,
|
||||||
|
],
|
||||||
|
|
||||||
'spaces' => [
|
'spaces' => [
|
||||||
'driver' => 's3',
|
'driver' => 's3',
|
||||||
'key' => env('DO_SPACES_KEY'),
|
'key' => env('DO_SPACES_KEY'),
|
||||||
|
|
|
||||||
|
|
@ -1,24 +1,26 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
return [
|
return [
|
||||||
'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
|
'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
|
||||||
|
|
||||||
'exif' => [
|
'exif' => [
|
||||||
'database' => env('MEDIA_EXIF_DATABASE', false),
|
'database' => env('MEDIA_EXIF_DATABASE', false),
|
||||||
],
|
],
|
||||||
|
|
||||||
'storage' => [
|
'storage' => [
|
||||||
'remote' => [
|
'remote' => [
|
||||||
/*
|
/*
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
| Store remote media on cloud/S3
|
| Store remote media on cloud/S3
|
||||||
|--------------------------------------------------------------------------
|
|--------------------------------------------------------------------------
|
||||||
|
|
|
|
||||||
| Set this to cache remote media on cloud/S3 filesystem drivers.
|
| Set this to cache remote media on cloud/S3 filesystem drivers.
|
||||||
| Disabled by default.
|
| Disabled by default.
|
||||||
|
|
|
|
||||||
*/
|
*/
|
||||||
'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false)
|
'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false),
|
||||||
],
|
|
||||||
]
|
'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false),
|
||||||
|
],
|
||||||
|
]
|
||||||
];
|
];
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue