diff --git a/app/Http/Controllers/PublicApiController.php b/app/Http/Controllers/PublicApiController.php index e27c39226..e6f3bf428 100644 --- a/app/Http/Controllers/PublicApiController.php +++ b/app/Http/Controllers/PublicApiController.php @@ -32,6 +32,7 @@ use App\Services\{ LikeService, PublicTimelineService, ProfileService, + NetworkTimelineService, ReblogService, RelationshipService, StatusService, @@ -608,59 +609,92 @@ class PublicApiController extends Controller $filtered = $user ? UserFilterService::filters($user->profile_id) : []; - if($min || $max) { - $dir = $min ? '>' : '<'; - $id = $min ?? $max; - $timeline = Status::select( - 'id', - 'uri', - 'type', - 'scope', - 'created_at', - ) - ->where('id', $dir, $id) - ->whereNull(['in_reply_to_id', 'reblog_of_id']) - ->whereNotIn('profile_id', $filtered) - ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) - ->whereNotNull('uri') - ->whereScope('public') - ->where('id', '>', $amin) - ->orderBy('created_at', 'desc') - ->limit($limit) - ->get() - ->map(function($s) use ($user) { - $status = StatusService::get($s->id); - $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id); - $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id); - $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id); - return $status; - }); - $res = $timeline->toArray(); - } else { - $timeline = Status::select( - 'id', - 'uri', - 'type', - 'scope', - 'created_at', - ) - ->whereNull(['in_reply_to_id', 'reblog_of_id']) - ->whereNotIn('profile_id', $filtered) - ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) - ->whereNotNull('uri') - ->whereScope('public') - ->where('id', '>', $amin) - ->orderBy('created_at', 'desc') - ->limit($limit) - ->get() - ->map(function($s) use ($user) { - $status = StatusService::get($s->id); - $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id); - $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id); - $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id); - return $status; - }); - $res = $timeline->toArray(); + if(config('instance.timeline.network.cached') == false) { + if($min || $max) { + $dir = $min ? '>' : '<'; + $id = $min ?? $max; + $timeline = Status::select( + 'id', + 'uri', + 'type', + 'scope', + 'created_at', + ) + ->where('id', $dir, $id) + ->whereNull(['in_reply_to_id', 'reblog_of_id']) + ->whereNotIn('profile_id', $filtered) + ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) + ->whereNotNull('uri') + ->whereScope('public') + ->where('id', '>', $amin) + ->orderBy('created_at', 'desc') + ->limit($limit) + ->get() + ->map(function($s) use ($user) { + $status = StatusService::get($s->id); + $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id); + $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id); + $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id); + return $status; + }); + $res = $timeline->toArray(); + } else { + $timeline = Status::select( + 'id', + 'uri', + 'type', + 'scope', + 'created_at', + ) + ->whereNull(['in_reply_to_id', 'reblog_of_id']) + ->whereNotIn('profile_id', $filtered) + ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) + ->whereNotNull('uri') + ->whereScope('public') + ->where('id', '>', $amin) + ->orderBy('created_at', 'desc') + ->limit($limit) + ->get() + ->map(function($s) use ($user) { + $status = StatusService::get($s->id); + $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id); + $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id); + $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id); + return $status; + }); + $res = $timeline->toArray(); + } + } else { + Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() { + if(NetworkTimelineService::count() == 0) { + NetworkTimelineService::warmCache(true, 400); + } + }); + + if ($max) { + $feed = NetworkTimelineService::getRankedMaxId($max, $limit); + } else if ($min) { + $feed = NetworkTimelineService::getRankedMinId($min, $limit); + } else { + $feed = NetworkTimelineService::get(0, $limit); + } + + $res = collect($feed) + ->map(function($k) use($user) { + $status = StatusService::get($k); + if($status && isset($status['account']) && $user) { + $status['favourited'] = (bool) LikeService::liked($user->profile_id, $k); + $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $k); + $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $k); + $status['relationship'] = RelationshipService::get($user->profile_id, $status['account']['id']); + } + return $status; + }) + ->filter(function($s) use($filtered) { + return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false; + }) + ->values() + ->toArray(); } return response()->json($res); diff --git a/app/Services/NetworkTimelineService.php b/app/Services/NetworkTimelineService.php new file mode 100644 index 000000000..959019b9a --- /dev/null +++ b/app/Services/NetworkTimelineService.php @@ -0,0 +1,95 @@ + 100) { + $stop = 100; + } + + return Redis::zrevrange(self::CACHE_KEY, $start, $stop); + } + + public static function getRankedMaxId($start = null, $limit = 10) + { + if(!$start) { + return []; + } + + return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, $start, '-inf', [ + 'withscores' => true, + 'limit' => [1, $limit] + ])); + } + + public static function getRankedMinId($end = null, $limit = 10) + { + if(!$end) { + return []; + } + + return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, '+inf', $end, [ + 'withscores' => true, + 'limit' => [0, $limit] + ])); + } + + public static function add($val) + { + if(self::count() > config('instance.timeline.network.cache_dropoff')) { + if(config('database.redis.client') === 'phpredis') { + Redis::zpopmin(self::CACHE_KEY); + } + } + + return Redis::zadd(self::CACHE_KEY, $val, $val); + } + + public static function rem($val) + { + return Redis::zrem(self::CACHE_KEY, $val); + } + + public static function del($val) + { + return self::rem($val); + } + + public static function count() + { + return Redis::zcard(self::CACHE_KEY); + } + + public static function warmCache($force = false, $limit = 100) + { + if(self::count() == 0 || $force == true) { + Redis::del(self::CACHE_KEY); + $ids = Status::whereNotNull('uri') + ->whereScope('public') + ->whereNull('in_reply_to_id') + ->whereNull('reblog_of_id') + ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) + ->where('created_at', '>', now()->subHours(config('instance.timeline.network.max_hours_old'))) + ->orderByDesc('created_at') + ->limit($limit) + ->pluck('id'); + foreach($ids as $id) { + self::add($id); + } + return 1; + } + return 0; + } +} diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index d471482b7..d058f2fd0 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -32,6 +32,7 @@ use App\Services\CustomEmojiService; use App\Services\InstanceService; use App\Services\MediaPathService; use App\Services\MediaStorageService; +use App\Services\NetworkTimelineService; use App\Jobs\MediaPipeline\MediaStoragePipeline; use App\Jobs\AvatarPipeline\RemoteAvatarFetch; use App\Util\Media\License; @@ -490,6 +491,16 @@ class Helpers { if(isset($activity['tag']) && is_array($activity['tag']) && !empty($activity['tag'])) { StatusTagsPipeline::dispatch($activity, $status); } + + if( config('instance.timeline.network.cached') && + $status->in_reply_to_id === null && + $status->reblog_of_id === null && + in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) && + $status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old')) + ) { + NetworkTimelineService::add($status->id); + } + return $status; }); } diff --git a/config/instance.php b/config/instance.php index 3e8e8c414..15ce670f8 100644 --- a/config/instance.php +++ b/config/instance.php @@ -24,6 +24,12 @@ return [ 'timeline' => [ 'local' => [ 'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false) + ], + + 'network' => [ + 'cached' => env('PF_NETWORK_TIMELINE') ? env('INSTANCE_NETWORK_TIMELINE_CACHED', false) : false, + 'cache_dropoff' => env('INSTANCE_NETWORK_TIMELINE_CACHE_DROPOFF', 100), + 'max_hours_old' => env('INSTANCE_NETWORK_TIMELINE_CACHE_MAX_HOUR_INGEST', 6) ] ],