chris/pixelfed
1
0
Fork 0
forked from mirror/pixelfed
Code Pull requests Activity

Update FollowServiceWarmCache, improve handling larger following/follower lists

Browse source
This commit is contained in:
Daniel Supernault 2023-08-23 03:45:51 -06:00
parent 93c7ad9779
commit 61a6d90403
No known key found for this signature in database
GPG key ID: 0DEF1C662C9033F7
2 changed files with 149 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
app/Jobs/FollowPipeline/FollowServiceWarmCache.php
View file

@ -8,10 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use App\Services\AccountService;
use App\Services\FollowerService;
use Cache;
use DB;
use Storage;
use App\Follower;
use App\Profile;
class FollowServiceWarmCache implements ShouldQueue
@ -23,6 +26,16 @@ class FollowServiceWarmCache implements ShouldQueue
public $timeout = 5000;
public $failOnTimeout = false;
/**
* Get the middleware the job should pass through.
*
* @return array<int, object>
*/
public function middleware(): array
{
return [(new WithoutOverlapping($this->profileId))->dontRelease()];
}
/**
* Create a new job instance.
*
@ -42,6 +55,10 @@ class FollowServiceWarmCache implements ShouldQueue
{
$id = $this->profileId;
if(Cache::has(FollowerService::FOLLOWERS_SYNC_KEY . $id) && Cache::has(FollowerService::FOLLOWING_SYNC_KEY . $id)) {
return;
}
$account = AccountService::get($id, true);
if(!$account) {
@ -50,25 +67,43 @@ class FollowServiceWarmCache implements ShouldQueue
return;
}
DB::table('followers')
->select('id', 'following_id', 'profile_id')
->whereFollowingId($id)
->orderBy('id')
->chunk(200, function($followers) use($id) {
foreach($followers as $follow) {
FollowerService::add($follow->profile_id, $id);
}
});
$hasFollowerPostProcessing = false;
$hasFollowingPostProcessing = false;
DB::table('followers')
->select('id', 'following_id', 'profile_id')
->whereProfileId($id)
->orderBy('id')
->chunk(200, function($followers) use($id) {
foreach($followers as $follow) {
FollowerService::add($id, $follow->following_id);
if(Follower::whereProfileId($id)->orWhere('following_id', $id)->count()) {
$following = [];
$followers = [];
foreach(Follower::lazy() as $follow) {
if($follow->following_id != $id && $follow->profile_id != $id) {
continue;
}
if($follow->profile_id == $id) {
$following[] = $follow->following_id;
} else {
$followers[] = $follow->profile_id;
}
}
if(count($followers) > 100) {
// store follower ids and process in another job
Storage::put('follow-warm-cache/' . $id . '/followers.json', json_encode($followers));
$hasFollowerPostProcessing = true;
} else {
foreach($followers as $follower) {
FollowerService::add($follower, $id);
}
}
if(count($following) > 100) {
// store following ids and process in another job
Storage::put('follow-warm-cache/' . $id . '/following.json', json_encode($following));
$hasFollowingPostProcessing = true;
} else {
foreach($following as $following) {
FollowerService::add($id, $following);
}
}
}
});
Cache::put(FollowerService::FOLLOWERS_SYNC_KEY . $id, 1, 604800);
Cache::put(FollowerService::FOLLOWING_SYNC_KEY . $id, 1, 604800);
@ -82,6 +117,14 @@ class FollowServiceWarmCache implements ShouldQueue
AccountService::del($id);
if($hasFollowingPostProcessing) {
FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'following')->onQueue('follow');
}
if($hasFollowerPostProcessing) {
FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'followers')->onQueue('follow');
}
return;
}
}

88
app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php Normal file
View file

@ -0,0 +1,88 @@
<?php
namespace App\Jobs\FollowPipeline;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Services\AccountService;
use App\Services\FollowerService;
use Cache;
use DB;
use Storage;
use App\Follower;
use App\Profile;
class FollowServiceWarmCacheLargeIngestPipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $profileId;
public $followType;
public $tries = 5;
public $timeout = 5000;
public $failOnTimeout = false;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($profileId, $followType = 'following')
{
$this->profileId = $profileId;
$this->followType = $followType;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$pid = $this->profileId;
$type = $this->followType;
if($type === 'followers') {
$key = 'follow-warm-cache/' . $pid . '/followers.json';
if(!Storage::exists($key)) {
return;
}
$file = Storage::get($key);
$json = json_decode($file, true);
foreach($json as $id) {
FollowerService::add($id, $pid, false);
usleep(random_int(500, 3000));
}
sleep(5);
Storage::delete($key);
}
if($type === 'following') {
$key = 'follow-warm-cache/' . $pid . '/following.json';
if(!Storage::exists($key)) {
return;
}
$file = Storage::get($key);
$json = json_decode($file, true);
foreach($json as $id) {
FollowerService::add($pid, $id, false);
usleep(random_int(500, 3000));
}
sleep(5);
Storage::delete($key);
}
sleep(random_int(2, 5));
$files = Storage::files('follow-warm-cache/' . $pid);
if(empty($files)) {
Storage::deleteDirectory('follow-warm-cache/' . $pid);
}
}
}