chris/pixelfed
1
0
Fork 0
forked from mirror/pixelfed
Code Pull requests Activity

Merge pull request #3875 from pixelfed/staging

Browse source 
Staging
This commit is contained in:
daniel 2022-12-02 00:23:31 -07:00 committed by GitHub
commit fdf28dd6ac
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 204 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
CHANGELOG.md
View file

@ -30,6 +30,9 @@
- Update InboxPipeline, bump request timeout from 5s to 60s ([bb120019](https://github.com/pixelfed/pixelfed/commit/bb120019))
- Update web routes, fix missing home route ([a9f4ddfc](https://github.com/pixelfed/pixelfed/commit/a9f4ddfc))
- Allow forceHttps to be disabled, fixes #3710 ([a31bdec7](https://github.com/pixelfed/pixelfed/commit/a31bdec7))
- Update MediaStorageService, fix size check bug ([319f0ba5](https://github.com/pixelfed/pixelfed/commit/319f0ba5))
- Update AvatarSync, fix sync skipping recently fetched avatars by setting last_fetched_at to null before refetching ([a83fc798](https://github.com/pixelfed/pixelfed/commit/a83fc798))
- Refactor AvatarStorage to support migrating avatars to cloud storage, fix remote avatar refetching and merge AvatarSync commands and add deprecation notice to avatar:sync command ([223aea47](https://github.com/pixelfed/pixelfed/commit/223aea47))
- ([](https://github.com/pixelfed/pixelfed/commit/))
## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)

2
app/Avatar.php
View file

@ -20,7 +20,7 @@ class Avatar extends Model
'last_processed_at'
];
protected $fillable = ['profile_id'];
protected $guarded = [];
protected $visible = [
'id',

178
app/Console/Commands/AvatarStorage.php
View file

@ -4,9 +4,14 @@ namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Avatar;
use App\Profile;
use App\User;
use Cache;
use Storage;
use App\Services\AccountService;
use App\Util\Lexer\PrettyNumber;
use Illuminate\Support\Str;
use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
class AvatarStorage extends Command
{
@ -24,6 +29,11 @@ class AvatarStorage extends Command
*/
protected $description = 'Manage avatar storage';
public $found = 0;
public $notFetched = 0;
public $fixed = 0;
public $missing = 0;
/**
* Execute the console command.
*
@ -90,13 +100,25 @@ class AvatarStorage extends Command
$this->info($msg);
}
$choice = $this->choice(
'Select action:',
$options = config_cache('pixelfed.cloud_storage') && config_cache('instance.avatar.local_to_cloud') ?
[
'Cancel',
'Upload default avatar to cloud',
'Move local avatars to cloud',
'Move cloud avatars to local'
],
'Re-fetch remote avatars'
] : [
'Cancel',
'Re-fetch remote avatars'
];
$this->missing = Profile::where('created_at', '<', now()->subDays(1))->doesntHave('avatar')->count();
if($this->missing != 0) {
$options[] = 'Fix missing avatars';
}
$choice = $this->choice(
'Select action:',
$options,
0
);
@ -106,18 +128,166 @@ class AvatarStorage extends Command
protected function handleChoice($id)
{
switch ($id) {
case 'Cancel':
return;
break;
case 'Upload default avatar to cloud':
return $this->uploadDefaultAvatar();
break;
case 'Move local avatars to cloud':
return $this->uploadAvatarsToCloud();
break;
case 'Re-fetch remote avatars':
return $this->refetchRemoteAvatars();
break;
case 'Fix missing avatars':
return $this->fixMissingAvatars();
break;
}
}
protected function uploadDefaultAvatar()
{
if(!$this->confirm('Are you sure you want to upload the default avatar to the cloud storage disk?')) {
return;
}
$disk = Storage::disk(config_cache('filesystems.cloud'));
$disk->put('cache/avatars/default.jpg', Storage::get('public/avatars/default.jpg'));
Avatar::whereMediaPath('public/avatars/default.jpg')->update(['cdn_url' => $disk->url('cache/avatars/default.jpg')]);
$this->info('Successfully uploaded default avatar to cloud storage!');
$this->info($disk->url('cache/avatars/default.jpg'));
}
protected function uploadAvatarsToCloud()
{
if(!config_cache('pixelfed.cloud_storage') || !config_cache('instance.avatar.local_to_cloud')) {
$this->error('Enable cloud storage and avatar cloud storage to perform this action');
return;
}
$confirm = $this->confirm('Are you sure you want to move local avatars to cloud storage?');
if(!$confirm) {
$this->error('Aborted action');
return;
}
$disk = Storage::disk(config_cache('filesystems.cloud'));
if($disk->missing('cache/avatars/default.jpg')) {
$disk->put('cache/avatars/default.jpg', Storage::get('public/avatars/default.jpg'));
}
Avatar::whereNull('is_remote')->chunk(5, function($avatars) use($disk) {
foreach($avatars as $avatar) {
if($avatar->media_path === 'public/avatars/default.jpg') {
$avatar->cdn_url = $disk->url('cache/avatars/default.jpg');
$avatar->save();
} else {
if(!$avatar->media_path || !Str::of($avatar->media_path)->startsWith('public/avatars/')) {
continue;
}
$ext = pathinfo($avatar->media_path, PATHINFO_EXTENSION);
$newPath = 'cache/avatars/' . $avatar->profile_id . '/avatar_' . strtolower(Str::random(6)) . '.' . $ext;
$existing = Storage::disk('local')->get($avatar->media_path);
if(!$existing) {
continue;
}
$newMediaPath = $disk->put($newPath, $existing);
$avatar->media_path = $newMediaPath;
$avatar->cdn_url = $disk->url($newMediaPath);
$avatar->save();
}
Cache::forget('avatar:' . $avatar->profile_id);
Cache::forget(AccountService::CACHE_KEY . $avatar->profile_id);
}
});
}
protected function refetchRemoteAvatars()
{
if(!$this->confirm('Are you sure you want to refetch all remote avatars? This could take a while.')) {
return;
}
if(config_cache('pixelfed.cloud_storage') == false && config_cache('federation.avatars.store_local') == false) {
$this->error('You have cloud storage disabled and local avatar storage disabled, we cannot refetch avatars.');
return;
}
$count = Profile::has('avatar')
->with('avatar')
->whereNull('user_id')
->count();
$this->info('Found ' . $count . ' remote avatars to re-fetch');
$this->line(' ');
$bar = $this->output->createProgressBar($count);
Profile::has('avatar')
->with('avatar')
->whereNull('user_id')
->chunk(50, function($profiles) use($bar) {
foreach($profiles as $profile) {
$avatar = $profile->avatar;
$avatar->last_fetched_at = null;
$avatar->save();
RemoteAvatarFetch::dispatch($profile)->onQueue('low');
$bar->advance();
}
});
$this->line(' ');
$this->line(' ');
$this->info('Finished dispatching avatar refetch jobs!');
$this->line(' ');
$this->info('This may take a few minutes to complete, you may need to run "php artisan cache:clear" after the jobs are processed.');
$this->line(' ');
}
protected function incr($name)
{
switch($name) {
case 'found':
$this->found = $this->found + 1;
break;
case 'notFetched':
$this->notFetched = $this->notFetched + 1;
break;
case 'fixed':
$this->fixed++;
break;
}
}
protected function fixMissingAvatars()
{
if(!$this->confirm('Are you sure you want to fix missing avatars?')) {
return;
}
$this->info('Found ' . $this->missing . ' accounts with missing profiles');
Profile::where('created_at', '<', now()->subDays(1))
->doesntHave('avatar')
->chunk(50, function($profiles) {
foreach($profiles as $profile) {
Avatar::updateOrCreate([
'profile_id' => $profile->id
], [
'media_path' => 'public/avatars/default.jpg',
'is_remote' => $profile->domain == null && $profile->private_key == null
]);
$this->incr('fixed');
}
});
$this->line(' ');
$this->line(' ');
$this->info('Fixed ' . $this->fixed . ' accounts with a blank avatar');
}
}

68
app/Console/Commands/AvatarSync.php
View file

@ -48,6 +48,18 @@ class AvatarSync extends Command
public function handle()
{
$this->info('Welcome to the avatar sync manager');
$this->line(' ');
$this->line(' ');
$this->error('This command is deprecated and will be removed in a future version');
$this->error('You should use the following command instead: ');
$this->line(' ');
$this->info('php artisan avatar:storage');
$this->line(' ');
$confirm = $this->confirm('Are you sure you want to use this deprecated command even though it is no longer supported?');
if(!$confirm) {
return;
}
$actions = [
'Analyze',
@ -123,7 +135,7 @@ class AvatarSync extends Command
$bar = $this->output->createProgressBar($count);
$bar->start();
Profile::chunk(5000, function($profiles) use ($bar) {
Profile::chunk(50, function($profiles) use ($bar) {
foreach($profiles as $profile) {
if($profile->domain == null) {
$bar->advance();
@ -146,41 +158,11 @@ class AvatarSync extends Command
protected function fetch()
{
$this->info('Fetching ....');
Avatar::whereIsRemote(true)
->whereNull('cdn_url')
// ->with('profile')
->chunk(10, function($avatars) {
foreach($avatars as $avatar) {
if(!$avatar || !$avatar->profile) {
continue;
}
$url = $avatar->profile->remote_url;
if(!$url || !Helpers::validateUrl($url)) {
continue;
}
try {
$res = Helpers::fetchFromUrl($url);
if(
!is_array($res) ||
!isset($res['@context']) ||
!isset($res['icon']) ||
!isset($res['icon']['type']) ||
!isset($res['icon']['url']) ||
!Str::endsWith($res['icon']['url'], ['.png', '.jpg', '.jpeg'])
) {
continue;
}
} catch (\GuzzleHttp\Exception\RequestException $e) {
continue;
} catch(\Illuminate\Http\Client\ConnectionException $e) {
continue;
}
$avatar->remote_url = $res['icon']['url'];
$avatar->save();
RemoteAvatarFetch::dispatch($avatar->profile);
}
});
$this->error('This action has been deprecated, please run the following command instead:');
$this->line(' ');
$this->info('php artisan avatar:storage');
$this->line(' ');
return;
}
protected function fix()
@ -208,12 +190,10 @@ class AvatarSync extends Command
protected function sync()
{
Avatar::whereIsRemote(true)
->with('profile')
->chunk(10, function($avatars) {
foreach($avatars as $avatar) {
RemoteAvatarFetch::dispatch($avatar->profile);
}
});
}
$this->error('This action has been deprecated, please run the following command instead:');
$this->line(' ');
$this->info('php artisan avatar:storage');
$this->line(' ');
return;
}
}

6
app/Services/MediaStorageService.php
View file

@ -12,6 +12,7 @@ use App\Media;
use App\Profile;
use App\User;
use GuzzleHttp\Client;
use App\Services\AccountService;
use App\Http\Controllers\AvatarController;
use GuzzleHttp\Exception\RequestException;
use App\Jobs\MediaPipeline\MediaDeletePipeline;
@ -226,10 +227,6 @@ class MediaStorageService {
return;
}
if($avatar->size && $head['length'] == $avatar->size) {
return;
}
$base = ($local ? 'public/cache/' : 'cache/') . 'avatars/' . $avatar->profile_id;
$ext = $head['mime'] == 'image/jpeg' ? 'jpg' : 'png';
$path = Str::random(20) . '_avatar.' . $ext;
@ -255,6 +252,7 @@ class MediaStorageService {
$avatar->save();
Cache::forget('avatar:' . $avatar->profile_id);
Cache::forget(AccountService::CACHE_KEY . $avatar->profile_id);
unlink($tmpName);
}