From 918016a5ad558f7dfca33fbd2d8369fe8c7d3ac0 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 6 Jul 2021 02:01:24 -0600 Subject: [PATCH 1/6] Update PublicTimelineService --- app/Services/PublicTimelineService.php | 41 ++++++++++++++++++++------ app/Services/StatusService.php | 3 +- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/app/Services/PublicTimelineService.php b/app/Services/PublicTimelineService.php index dac552f0e..c802269a5 100644 --- a/app/Services/PublicTimelineService.php +++ b/app/Services/PublicTimelineService.php @@ -18,18 +18,41 @@ class PublicTimelineService { if($stop > 100) { $stop = 100; } - $tl = []; - $keys = Redis::zrevrange(self::CACHE_KEY, $start, $stop); - foreach($keys as $key) { - array_push($tl, StatusService::get($key)); + + return Redis::zrevrange(self::CACHE_KEY, $start, $stop); + } + + public static function getRankedMaxId($start = null, $limit = 10) + { + if(!$start) { + return []; } - return $tl; + + return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, $start, '-inf', [ + 'withscores' => true, + 'limit' => [1, $limit] + ])); + } + + public static function getRankedMinId($end = null, $limit = 10) + { + if(!$end) { + return []; + } + + return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, '+inf', $end, [ + 'withscores' => true, + 'limit' => [0, $limit] + ])); } public static function add($val) { - // return Redis::zadd(self::CACHE_KEY, $val, $val); - return; + if(self::count() > 400) { + Redis::zpopmin(self::CACHE_KEY); + } + + return Redis::zadd(self::CACHE_KEY, $val, $val); } public static function rem($val) @@ -44,7 +67,7 @@ class PublicTimelineService { public static function count() { - return Redis::zcount(self::CACHE_KEY, '-inf', '+inf'); + return Redis::zcard(self::CACHE_KEY); } public static function warmCache($force = false, $limit = 100) @@ -55,7 +78,7 @@ class PublicTimelineService { ->whereNull('reblog_of_id') ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) ->whereScope('public') - ->latest() + ->orderByDesc('id') ->limit($limit) ->pluck('id'); foreach($ids as $id) { diff --git a/app/Services/StatusService.php b/app/Services/StatusService.php index 360fde852..8807e37b1 100644 --- a/app/Services/StatusService.php +++ b/app/Services/StatusService.php @@ -37,6 +37,7 @@ class StatusService { public static function del($id) { + PublicTimelineService::rem($id); return Cache::forget(self::key($id)); } -} \ No newline at end of file +} From 76b6ec2aa6a3fef759a03ff97f5b6026486b1e01 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 6 Jul 2021 19:36:43 -0600 Subject: [PATCH 2/6] Update admin instance view --- resources/views/admin/instances/home.blade.php | 1 + resources/views/admin/instances/show.blade.php | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/resources/views/admin/instances/home.blade.php b/resources/views/admin/instances/home.blade.php index 5bd16119d..c8b123da3 100644 --- a/resources/views/admin/instances/home.blade.php +++ b/resources/views/admin/instances/home.blade.php @@ -85,6 +85,7 @@ @endsection @push('scripts') + -@endpush \ No newline at end of file +@endpush From d43e6d8d0726413f745ca3d2938e7394210c3a9a Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 6 Jul 2021 20:10:42 -0600 Subject: [PATCH 3/6] Update NotificationService, use zrevrangebyscore for api --- app/Http/Controllers/Api/ApiV1Controller.php | 34 +++++------ app/Services/NotificationService.php | 61 +++++++++++++++++++- 2 files changed, 71 insertions(+), 24 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index c8bf8c702..e651bb204 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -1283,7 +1283,6 @@ class ApiV1Controller extends Controller $pid = $request->user()->profile_id; $limit = $request->input('limit', 20); - $timeago = now()->subMonths(6); $since = $request->input('since_id'); $min = $request->input('min_id'); @@ -1293,27 +1292,20 @@ class ApiV1Controller extends Controller $min = 1; } - $dir = $since ? '>' : ($min ? '>=' : '<'); - $id = $since ?? $min ?? $max; + $maxId = null; + $minId = null; - $notifications = Notification::whereProfileId($pid) - ->where('id', $dir, $id) - ->whereDate('created_at', '>', $timeago) - ->orderByDesc('id') - ->limit($limit) - ->get(); - - $minId = $notifications->min('id'); - $maxId = $notifications->max('id'); - - $resource = new Fractal\Resource\Collection( - $notifications, - new NotificationTransformer() - ); - - $res = $this->fractal - ->createData($resource) - ->toArray(); + if($max) { + $res = NotificationService::getMax($pid, $max, $limit); + $ids = NotificationService::getRankedMaxId($pid, $max, $limit); + $maxId = max($ids); + $minId = min($ids); + } else { + $res = NotificationService::getMin($pid, $min ?? $since, $limit); + $ids = NotificationService::getRankedMinId($pid, $min ?? $since, $limit); + $maxId = max($ids); + $minId = min($ids); + } $baseUrl = config('app.url') . '/api/v1/notifications?'; diff --git a/app/Services/NotificationService.php b/app/Services/NotificationService.php index bbf7a34f3..3de1bf650 100644 --- a/app/Services/NotificationService.php +++ b/app/Services/NotificationService.php @@ -46,6 +46,60 @@ class NotificationService { return $ids; } + public static function getMax($id = false, $start, $limit = 10) + { + $ids = self::getRankedMaxId($id, $start, $limit); + + if(empty($ids)) { + return []; + } + + $res = collect([]); + foreach($ids as $id) { + $res->push(self::getNotification($id)); + } + return $res->toArray(); + } + + public static function getMin($id = false, $start, $limit = 10) + { + $ids = self::getRankedMinId($id, $start, $limit); + + if(empty($ids)) { + return []; + } + + $res = collect([]); + foreach($ids as $id) { + $res->push(self::getNotification($id)); + } + return $res->toArray(); + } + + public static function getRankedMaxId($id = false, $start = null, $limit = 10) + { + if(!$start || !$id) { + return []; + } + + return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, $start, '-inf', [ + 'withscores' => true, + 'limit' => [1, $limit] + ])); + } + + public static function getRankedMinId($id = false, $end = null, $limit = 10) + { + if(!$end || !$id) { + return []; + } + + return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, '+inf', $end, [ + 'withscores' => true, + 'limit' => [0, $limit] + ])); + } + public static function set($id, $val) { return Redis::zadd(self::CACHE_KEY . $id, $val, $val); @@ -53,6 +107,7 @@ class NotificationService { public static function del($id, $val) { + Cache::forget('service:notification:' . $val); return Redis::zrem(self::CACHE_KEY . $id, $val); } @@ -73,7 +128,7 @@ class NotificationService { public static function getNotification($id) { - return Cache::remember('service:notification:'.$id, now()->addMonths(3), function() use($id) { + return Cache::remember('service:notification:'.$id, now()->addDays(3), function() use($id) { $n = Notification::with('item')->findOrFail($id); $fractal = new Fractal\Manager(); $fractal->setSerializer(new ArraySerializer()); @@ -84,7 +139,7 @@ class NotificationService { public static function setNotification(Notification $notification) { - return Cache::remember('service:notification:'.$notification->id, now()->addMonths(3), function() use($notification) { + return Cache::remember('service:notification:'.$notification->id, now()->addDays(3), function() use($notification) { $fractal = new Fractal\Manager(); $fractal->setSerializer(new ArraySerializer()); $resource = new Fractal\Resource\Item($notification, new NotificationTransformer()); @@ -106,4 +161,4 @@ class NotificationService { } return 0; } -} \ No newline at end of file +} From f67c67bce10fb142f23281444f59792312c2c1f4 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 6 Jul 2021 20:29:52 -0600 Subject: [PATCH 4/6] Update ApiV1Controller, use PublicTimelineService --- app/Http/Controllers/Api/ApiV1Controller.php | 89 +++++--------------- 1 file changed, 19 insertions(+), 70 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index e651bb204..583854229 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -49,8 +49,10 @@ use App\Jobs\VideoPipeline\{ VideoThumbnail }; use App\Services\{ + LikeService, NotificationService, MediaPathService, + PublicTimelineService, SearchApiV2Service, StatusService, MediaBlocklistService @@ -1462,90 +1464,37 @@ class ApiV1Controller extends Controller public function timelinePublic(Request $request) { $this->validate($request,[ - 'page' => 'nullable|integer|max:40', 'min_id' => 'nullable|integer|min:0|max:' . PHP_INT_MAX, 'max_id' => 'nullable|integer|min:0|max:' . PHP_INT_MAX, 'limit' => 'nullable|integer|max:80' ]); - $page = $request->input('page'); $min = $request->input('min_id'); $max = $request->input('max_id'); $limit = $request->input('limit') ?? 3; $user = $request->user(); - if($user) { - $key = 'user:last_active_at:id:'.$user->id; - $ttl = now()->addMinutes(5); - Cache::remember($key, $ttl, function() use($user) { - $user->last_active_at = now(); - $user->save(); - return; - }); - } + if(PublicTimelineService::count() == 0) { + PublicTimelineService::warmCache(true, 400); + } - if($min || $max) { - $dir = $min ? '>' : '<'; - $id = $min ?? $max; - $timeline = Status::select( - 'id', - 'uri', - 'caption', - 'rendered', - 'profile_id', - 'type', - 'in_reply_to_id', - 'reblog_of_id', - 'is_nsfw', - 'scope', - 'local', - 'reply_count', - 'comments_disabled', - 'place_id', - 'likes_count', - 'reblogs_count', - 'created_at', - 'updated_at' - )->whereNull('uri') - ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album']) - ->with('profile', 'hashtags', 'mentions') - ->where('id', $dir, $id) - ->whereScope('public') - ->where('created_at', '>', now()->subDays(14)) - ->latest() - ->limit($limit) - ->get(); + if ($max) { + $feed = PublicTimelineService::getRankedMaxId($max, $limit); + } else if ($min) { + $feed = PublicTimelineService::getRankedMinId($min, $limit); } else { - $timeline = Status::select( - 'id', - 'uri', - 'caption', - 'rendered', - 'profile_id', - 'type', - 'in_reply_to_id', - 'reblog_of_id', - 'is_nsfw', - 'scope', - 'local', - 'reply_count', - 'comments_disabled', - 'place_id', - 'likes_count', - 'reblogs_count', - 'created_at', - 'updated_at' - )->whereNull('uri') - ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album']) - ->with('profile', 'hashtags', 'mentions') - ->whereScope('public') - ->where('created_at', '>', now()->subDays(14)) - ->latest() - ->simplePaginate($limit); + $feed = PublicTimelineService::get(0, $limit); } - $fractal = new Fractal\Resource\Collection($timeline, new StatusTransformer()); - $res = $this->fractal->createData($fractal)->toArray(); + $res = collect($feed) + ->map(function($k) use($user) { + $status = StatusService::get($k); + if($user) { + $status['favourited'] = (bool) LikeService::liked($user->profile_id, $k); + } + return $status; + }) + ->toArray(); return response()->json($res); } From 352aa573460a0cbad85a62bbd7296a475199183f Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 6 Jul 2021 20:43:24 -0600 Subject: [PATCH 5/6] Update ApiV1Controller, use ProfileService for verify_credentials --- app/Http/Controllers/Api/ApiV1Controller.php | 21 +++++--------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index 583854229..26bb0dd49 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -53,6 +53,7 @@ use App\Services\{ NotificationService, MediaPathService, PublicTimelineService, + ProfileService, SearchApiV2Service, StatusService, MediaBlocklistService @@ -118,25 +119,13 @@ class ApiV1Controller extends Controller public function verifyCredentials(Request $request) { abort_if(!$request->user(), 403); - $id = $request->user()->id; + $id = $request->user()->profile_id; - if($request->user()->last_active_at) { - $key = 'user:last_active_at:id:'.$id; - $ttl = now()->addMinutes(5); - Cache::remember($key, $ttl, function() use($id) { - $user = User::findOrFail($id); - $user->last_active_at = now(); - $user->save(); - return; - }); - } + $res = ProfileService::get($id); - $profile = Profile::whereNull('status')->whereUserId($id)->firstOrFail(); - $resource = new Fractal\Resource\Item($profile, new AccountTransformer()); - $res = $this->fractal->createData($resource)->toArray(); $res['source'] = [ - 'privacy' => $profile->is_private ? 'private' : 'public', - 'sensitive' => $profile->cw ? true : false, + 'privacy' => $res['locked'] ? 'private' : 'public', + 'sensitive' => false, 'language' => null, 'note' => '', 'fields' => [] From 3a0d88e2b4d806fe06ddd429e31e174cb7173a36 Mon Sep 17 00:00:00 2001 From: Daniel Supernault Date: Tue, 6 Jul 2021 20:43:53 -0600 Subject: [PATCH 6/6] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3feb5b68..d2758166f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,9 @@ - Updated PublicApiController, add LikeService to Network timeline. ([82895591](https://github.com/pixelfed/pixelfed/commit/82895591)) - Updated moderator api, expire cached status in StatusService. ([f215ee26](https://github.com/pixelfed/pixelfed/commit/f215ee26)) - Updated StatusHashtagService, fix null status bug. ([51a277e1](https://github.com/pixelfed/pixelfed/commit/51a277e1)) +- Updated NotificationService, use zrevrangebyscore for api. ([d43e6d8d](https://github.com/pixelfed/pixelfed/commit/d43e6d8d)) +- Updated ApiV1Controller, use PublicTimelineService. ([f67c67bc](https://github.com/pixelfed/pixelfed/commit/f67c67bc)) +- Updated ApiV1Controller, use ProfileService for verify_credentials. ([352aa573](https://github.com/pixelfed/pixelfed/commit/352aa573)) - ([](https://github.com/pixelfed/pixelfed/commit/)) ## [v0.11.0 (2021-06-01)](https://github.com/pixelfed/pixelfed/compare/v0.10.10...v0.11.0)