From 9174d83b64b624fa5a8c09f177002a3a3419000e Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 20 May 2022 20:36:35 -0600 Subject: [PATCH 1/5] Update migration, fixes #3497 --- .../migrations/2022_04_20_061915_create_conversations_table.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/database/migrations/2022_04_20_061915_create_conversations_table.php b/database/migrations/2022_04_20_061915_create_conversations_table.php index e94b6fdc..f4ac3596 100644 --- a/database/migrations/2022_04_20_061915_create_conversations_table.php +++ b/database/migrations/2022_04_20_061915_create_conversations_table.php @@ -16,6 +16,8 @@ class CreateConversationsTable extends Migration */ public function up() { + Schema::dropIfExists('conversations'); + Schema::create('conversations', function (Blueprint $table) { $table->bigIncrements('id'); $table->bigInteger('to_id')->unsigned()->index(); From 108e380394f32e7b2829e8aa9ec1e23a35818359 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 20 May 2022 21:17:54 -0600 Subject: [PATCH 2/5] Add ffmpeg config, disable logging by default --- config/laravel-ffmpeg.php | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 config/laravel-ffmpeg.php diff --git a/config/laravel-ffmpeg.php b/config/laravel-ffmpeg.php new file mode 100644 index 00000000..21315f19 --- /dev/null +++ b/config/laravel-ffmpeg.php @@ -0,0 +1,21 @@ + [ + 'binaries' => env('FFMPEG_BINARIES', 'ffmpeg'), + + 'threads' => 12, // set to false to disable the default 'threads' filter + ], + + 'ffprobe' => [ + 'binaries' => env('FFPROBE_BINARIES', 'ffprobe'), + ], + + 'timeout' => 3600, + + 'enable_logging' => env('FFMPEG_LOG', false), + + 'set_command_and_error_output_on_exception' => false, + + 'temporary_files_root' => env('FFMPEG_TEMPORARY_FILES_ROOT', sys_get_temp_dir()), +]; From 505261dab179c1e1c3d49cb1a86952e557e18fe4 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 20 May 2022 23:09:25 -0600 Subject: [PATCH 3/5] Refactor AP profileFetch logic to fix race conditions and improve updating fields and avatars --- app/Profile.php | 2 +- app/Util/ActivityPub/Helpers.php | 111 ++++++++++++++++++------------- 2 files changed, 66 insertions(+), 47 deletions(-) diff --git a/app/Profile.php b/app/Profile.php index 17063ab8..85eeae01 100644 --- a/app/Profile.php +++ b/app/Profile.php @@ -26,7 +26,7 @@ class Profile extends Model ]; protected $hidden = ['private_key']; protected $visible = ['id', 'user_id', 'username', 'name']; - protected $fillable = ['user_id']; + protected $guarded = []; public function user() { diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index e24613e2..9a1aa274 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -36,6 +36,7 @@ use App\Jobs\MediaPipeline\MediaStoragePipeline; use App\Jobs\AvatarPipeline\RemoteAvatarFetch; use App\Util\Media\License; use App\Models\Poll; +use Illuminate\Contracts\Cache\LockTimeoutException; class Helpers { @@ -664,26 +665,43 @@ class Helpers { return; } - public static function profileFirstOrNew($url, $runJobs = false) + public static function profileFirstOrNew($url) { $url = self::validateUrl($url); - if($url == false || strlen($url) > 190) { + if($url == false) { return; } + + $host = parse_url($url, PHP_URL_HOST); + $local = config('pixelfed.domain.app') == $host ? true : false; + + if($local == true) { + $id = last(explode('/', $url)); + return Profile::whereNull('status') + ->whereNull('domain') + ->whereUsername($id) + ->firstOrFail(); + } + + if($profile = Profile::whereRemoteUrl($url)->first()) { + if($profile->last_fetched_at->lt(now()->subHours(24))) { + return self::profileUpdateOrCreate($url); + } + return $profile; + } + + return self::profileUpdateOrCreate($url); + } + + public static function profileUpdateOrCreate($url) + { $hash = base64_encode($url); $key = 'ap:profile:by_url:' . $hash; - $ttl = now()->addSeconds(60); - $profile = Cache::remember($key, $ttl, function() use($url, $runJobs) { - $host = parse_url($url, PHP_URL_HOST); - $local = config('pixelfed.domain.app') == $host ? true : false; + $lock = Cache::lock($key, 30); + $profile = null; - if($local == true) { - $id = last(explode('/', $url)); - return Profile::whereNull('status') - ->whereNull('domain') - ->whereUsername($id) - ->firstOrFail(); - } + try { + $lock->block(5); $res = self::fetchProfileFromUrl($url); if(isset($res['id']) == false) { @@ -703,47 +721,48 @@ class Helpers { abort_if(!self::validateUrl($res['inbox']), 400); abort_if(!self::validateUrl($res['id']), 400); - $profile = Profile::whereRemoteUrl($res['id'])->first(); - if(!$profile) { - $instance = Instance::firstOrCreate([ + $profile = DB::transaction(function() use($domain, $webfinger, $res, $runJobs) { + $instance = Instance::updateOrCreate([ 'domain' => $domain ]); if($instance->wasRecentlyCreated == true) { \App\Jobs\InstancePipeline\FetchNodeinfoPipeline::dispatch($instance)->onQueue('low'); } - $profile = DB::transaction(function() use($domain, $webfinger, $res, $runJobs) { - $profile = new Profile(); - $profile->domain = strtolower($domain); - $profile->username = Purify::clean($webfinger); - $profile->name = isset($res['name']) ? Purify::clean($res['name']) : 'user'; - $profile->bio = isset($res['summary']) ? Purify::clean($res['summary']) : null; - $profile->sharedInbox = isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null; - $profile->inbox_url = $res['inbox']; - $profile->outbox_url = isset($res['outbox']) ? $res['outbox'] : null; - $profile->remote_url = $res['id']; - $profile->public_key = $res['publicKey']['publicKeyPem']; - $profile->key_id = $res['publicKey']['id']; - $profile->webfinger = Purify::clean($webfinger); - $profile->last_fetched_at = now(); - $profile->save(); - RemoteAvatarFetch::dispatch($profile); - return $profile; - }); - } else { - // Update info after 24 hours - if($profile->last_fetched_at == null || - $profile->last_fetched_at->lt(now()->subHours(24)) == true + + $profile = Profile::updateOrCreate( + [ + 'domain' => strtolower($domain), + 'username' => Purify::clean($webfinger), + 'remote_url' => $res['id'], + ], + [ + 'name' => isset($res['name']) ? Purify::clean($res['name']) : 'user', + 'bio' => isset($res['summary']) ? Purify::clean($res['summary']) : null, + 'sharedInbox' => isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null, + 'inbox_url' => $res['inbox'], + 'outbox_url' => isset($res['outbox']) ? $res['outbox'] : null, + 'public_key' => $res['publicKey']['publicKeyPem'], + 'key_id' => $res['publicKey']['id'], + 'webfinger' => Purify::clean($webfinger), + ] + ); + + if( $profile->last_fetched_at == null || + $profile->last_fetched_at->lt(now()->subHours(24)) ) { - $profile->name = isset($res['name']) ? Purify::clean($res['name']) : 'user'; - $profile->bio = isset($res['summary']) ? Purify::clean($res['summary']) : null; - $profile->last_fetched_at = now(); - $profile->sharedInbox = isset($res['endpoints']) && isset($res['endpoints']['sharedInbox']) && Helpers::validateUrl($res['endpoints']['sharedInbox']) ? $res['endpoints']['sharedInbox'] : null; - $profile->save(); + RemoteAvatarFetch::dispatch($profile); } - RemoteAvatarFetch::dispatch($profile); - } + $profile->last_fetched_at = now(); + $profile->save(); + return $profile; + }); + return $profile; - }); + } catch (LockTimeoutException $e) { + } finally { + optional($lock)->release(); + } + return $profile; } From c989b2a54890ca0bbabf1a7bea9a615f10018a59 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 20 May 2022 23:11:53 -0600 Subject: [PATCH 4/5] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1de4f6a7..74116431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ - Update adminReportController, fix mail verification request 500 bug by changing filter precedence to catch deleted users that may still be cached in AccountService ([3f322e29](https://github.com/pixelfed/pixelfed/commit/3f322e29)) - Update AP Helpers, fix getSensitive and getScope missing parameters ([657c66c1](https://github.com/pixelfed/pixelfed/commit/657c66c1)) - Fix mastodon api compatibility ([#3499](https://github.com/pixelfed/pixelfed/pull/3499)) +- Add ffmpeg config, disable logging by default ([108e3803](https://github.com/pixelfed/pixelfed/commit/108e3803)) +- Refactor AP profileFetch logic to fix race conditions and improve updating fields and avatars ([505261da](https://github.com/pixelfed/pixelfed/commit/505261da)) +- ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.3 (2022-05-09)](https://github.com/pixelfed/pixelfed/compare/v0.11.2...v0.11.3) From 20e0371c1934f19d52dc1de1e632adac8df330b3 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 20 May 2022 23:19:45 -0600 Subject: [PATCH 5/5] Update AP helpers, remove unused variable --- app/Util/ActivityPub/Helpers.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 9a1aa274..d471482b 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -721,7 +721,7 @@ class Helpers { abort_if(!self::validateUrl($res['inbox']), 400); abort_if(!self::validateUrl($res['id']), 400); - $profile = DB::transaction(function() use($domain, $webfinger, $res, $runJobs) { + $profile = DB::transaction(function() use($domain, $webfinger, $res) { $instance = Instance::updateOrCreate([ 'domain' => $domain ]);