From da6943daeda885db514ce3115d2d2e513c64a99c Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Fri, 3 Sep 2021 20:45:56 -0600 Subject: [PATCH] Add InstancePipeline and NodeinfoService --- app/Console/Commands/StoryGC.php | 107 +++++------------- .../FetchNodeinfoPipeline.php | 56 +++++++++ .../InstanceCrawlPipeline.php | 43 +++++++ app/Services/NodeinfoService.php | 76 +++++++++++++ 4 files changed, 206 insertions(+), 76 deletions(-) create mode 100644 app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php create mode 100644 app/Jobs/InstancePipeline/InstanceCrawlPipeline.php create mode 100644 app/Services/NodeinfoService.php diff --git a/app/Console/Commands/StoryGC.php b/app/Console/Commands/StoryGC.php index 0ef8ba7f5..8dd0aefce 100644 --- a/app/Console/Commands/StoryGC.php +++ b/app/Console/Commands/StoryGC.php @@ -7,6 +7,9 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Storage; use App\Story; use App\StoryView; +use App\Jobs\StoryPipeline\StoryExpire; +use App\Jobs\StoryPipeline\StoryRotateMedia; +use App\Services\StoryService; class StoryGC extends Command { @@ -41,89 +44,41 @@ class StoryGC extends Command */ public function handle() { - $this->directoryScan(); - $this->deleteViews(); - $this->deleteStories(); + $this->archiveExpiredStories(); + $this->rotateMedia(); } - protected function directoryScan() + protected function archiveExpiredStories() { - $day = now()->day; + $stories = Story::whereActive(true) + ->where('expires_at', '<', now()) + ->get(); - if($day !== 3) { + foreach($stories as $story) { + StoryExpire::dispatch($story)->onQueue('story'); + } + } + + protected function rotateMedia() + { + $queue = StoryService::rotateQueue(); + + if(!$queue || count($queue) == 0) { return; } - $monthHash = substr(hash('sha1', date('Y').date('m')), 0, 12); - - $t1 = Storage::directories('public/_esm.t1'); - $t2 = Storage::directories('public/_esm.t2'); - - $dirs = array_merge($t1, $t2); - - foreach($dirs as $dir) { - $hash = last(explode('/', $dir)); - if($hash != $monthHash) { - $this->info('Found directory to delete: ' . $dir); - $this->deleteDirectory($dir); - } - } - - $mh = hash('sha256', date('Y').'-.-'.date('m')); - $monthHash = date('Y').date('m').substr($mh, 0, 6).substr($mh, 58, 6); - $dirs = Storage::directories('public/_esm.t3'); - - foreach($dirs as $dir) { - $hash = last(explode('/', $dir)); - if($hash != $monthHash) { - $this->info('Found directory to delete: ' . $dir); - $this->deleteDirectory($dir); - } - } - } - - protected function deleteDirectory($path) - { - Storage::deleteDirectory($path); - } - - protected function deleteViews() - { - StoryView::where('created_at', '<', now()->subMinutes(1441))->delete(); - } - - protected function deleteStories() - { - $stories = Story::where('created_at', '>', now()->subMinutes(30)) - ->whereNull('active') - ->get(); - - foreach($stories as $story) { - if(Storage::exists($story->path) == true) { - Storage::delete($story->path); - } - DB::transaction(function() use($story) { - StoryView::whereStoryId($story->id)->delete(); - $story->delete(); + collect($queue) + ->each(function($id) { + $story = StoryService::getById($id); + if(!$story) { + StoryService::removeRotateQueue($id); + return; + } + if($story->created_at->gt(now()->subMinutes(20))) { + return; + } + StoryRotateMedia::dispatch($story)->onQueue('story'); + StoryService::removeRotateQueue($id); }); - } - - $stories = Story::where('created_at', '<', now() - ->subMinutes(1441)) - ->get(); - - if($stories->count() == 0) { - exit; - } - - foreach($stories as $story) { - if(Storage::exists($story->path) == true) { - Storage::delete($story->path); - } - DB::transaction(function() use($story) { - StoryView::whereStoryId($story->id)->delete(); - $story->delete(); - }); - } } } diff --git a/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php b/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php new file mode 100644 index 000000000..b8c79d67f --- /dev/null +++ b/app/Jobs/InstancePipeline/FetchNodeinfoPipeline.php @@ -0,0 +1,56 @@ +instance = $instance; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $instance = $this->instance; + + $ni = NodeinfoService::get($instance->domain); + if($ni) { + if(isset($ni['software']) && is_array($ni['software']) && isset($ni['software']['name'])) { + $software = $ni['software']['name']; + $instance->software = strtolower(strip_tags($software)); + $instance->last_crawled_at = now(); + $instance->user_count = Profile::whereDomain($instance->domain)->count(); + $instance->save(); + } + } else { + $instance->user_count = Profile::whereDomain($instance->domain)->count(); + $instance->last_crawled_at = now(); + $instance->save(); + } + } +} diff --git a/app/Jobs/InstancePipeline/InstanceCrawlPipeline.php b/app/Jobs/InstancePipeline/InstanceCrawlPipeline.php new file mode 100644 index 000000000..f45355f9c --- /dev/null +++ b/app/Jobs/InstancePipeline/InstanceCrawlPipeline.php @@ -0,0 +1,43 @@ +whereNull('software')->chunk(50, function($instances) use($headers) { + foreach($instances as $instance) { + FetchNodeinfoPipeline::dispatch($instance)->onQueue('low'); + } + }); + } +} diff --git a/app/Services/NodeinfoService.php b/app/Services/NodeinfoService.php new file mode 100644 index 000000000..10575ff9f --- /dev/null +++ b/app/Services/NodeinfoService.php @@ -0,0 +1,76 @@ + 'application/json', + 'User-Agent' => "(Pixelfed/{$version}; +{$appUrl})", + ]; + + $url = 'https://' . $domain; + $wk = $url . '/.well-known/nodeinfo'; + + try { + $res = Http::withHeaders($headers) + ->timeout(5) + ->get($wk); + } catch (RequestException $e) { + return false; + } catch (ConnectionException $e) { + return false; + } catch (\Exception $e) { + return false; + } + + if(!$res) { + return false; + } + + $json = $res->json(); + + if( !isset($json['links'])) { + return false; + } + + if(is_array($json['links'])) { + if(isset($json['links']['href'])) { + $href = $json['links']['href']; + } else { + $href = $json['links'][0]['href']; + } + } else { + return false; + } + + $domain = parse_url($url, PHP_URL_HOST); + $hrefDomain = parse_url($href, PHP_URL_HOST); + + if($domain !== $hrefDomain) { + return 60; + } + + try { + $res = Http::withHeaders($headers) + ->timeout(5) + ->get($href); + } catch (RequestException $e) { + return false; + } catch (ConnectionException $e) { + return false; + } catch (\Exception $e) { + return false; + } + return $res->json(); + } +}