chris
/
pixelfed
forked from mirror/pixelfed
1
0
Fork 0
Code Pull Requests Activity

Merge pull request #2842 from pixelfed/staging 

API perf improvements
This commit is contained in:
daniel 2021-07-06 20:45:55 -06:00 committed by GitHub
parent 2ee4b5f853 3a0d88e2b4
commit d2545b2850
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 135 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -41,6 +41,9 @@
- Updated PublicApiController, add LikeService to Network timeline. ([82895591](https://github.com/pixelfed/pixelfed/commit/82895591))
- Updated moderator api, expire cached status in StatusService. ([f215ee26](https://github.com/pixelfed/pixelfed/commit/f215ee26))
- Updated StatusHashtagService, fix null status bug. ([51a277e1](https://github.com/pixelfed/pixelfed/commit/51a277e1))
- Updated NotificationService, use zrevrangebyscore for api. ([d43e6d8d](https://github.com/pixelfed/pixelfed/commit/d43e6d8d))
- Updated ApiV1Controller, use PublicTimelineService. ([f67c67bc](https://github.com/pixelfed/pixelfed/commit/f67c67bc))
- Updated ApiV1Controller, use ProfileService for verify_credentials. ([352aa573](https://github.com/pixelfed/pixelfed/commit/352aa573))
- ([](https://github.com/pixelfed/pixelfed/commit/))
## [v0.11.0 (2021-06-01)](https://github.com/pixelfed/pixelfed/compare/v0.10.10...v0.11.0)

144
app/Http/Controllers/Api/ApiV1Controller.php
View File

@ -49,8 +49,11 @@ use App\Jobs\VideoPipeline\{
VideoThumbnail
};
use App\Services\{
LikeService,
NotificationService,
MediaPathService,
PublicTimelineService,
ProfileService,
SearchApiV2Service,
StatusService,
MediaBlocklistService
@ -116,25 +119,13 @@ class ApiV1Controller extends Controller
public function verifyCredentials(Request $request)
{
abort_if(!$request->user(), 403);
$id = $request->user()->id;
$id = $request->user()->profile_id;
if($request->user()->last_active_at) {
$key = 'user:last_active_at:id:'.$id;
$ttl = now()->addMinutes(5);
Cache::remember($key, $ttl, function() use($id) {
$user = User::findOrFail($id);
$user->last_active_at = now();
$user->save();
return;
});
}
$res = ProfileService::get($id);
$profile = Profile::whereNull('status')->whereUserId($id)->firstOrFail();
$resource = new Fractal\Resource\Item($profile, new AccountTransformer());
$res = $this->fractal->createData($resource)->toArray();
$res['source'] = [
'privacy' => $profile->is_private ? 'private' : 'public',
'sensitive' => $profile->cw ? true : false,
'privacy' => $res['locked'] ? 'private' : 'public',
'sensitive' => false,
'language' => null,
'note' => '',
'fields' => []
@ -1283,7 +1274,6 @@ class ApiV1Controller extends Controller
$pid = $request->user()->profile_id;
$limit = $request->input('limit', 20);
$timeago = now()->subMonths(6);
$since = $request->input('since_id');
$min = $request->input('min_id');
@ -1293,27 +1283,20 @@ class ApiV1Controller extends Controller
$min = 1;
}
$dir = $since ? '>' : ($min ? '>=' : '<');
$id = $since ?? $min ?? $max;
$maxId = null;
$minId = null;
$notifications = Notification::whereProfileId($pid)
->where('id', $dir, $id)
->whereDate('created_at', '>', $timeago)
->orderByDesc('id')
->limit($limit)
->get();
$minId = $notifications->min('id');
$maxId = $notifications->max('id');
$resource = new Fractal\Resource\Collection(
$notifications,
new NotificationTransformer()
);
$res = $this->fractal
->createData($resource)
->toArray();
if($max) {
$res = NotificationService::getMax($pid, $max, $limit);
$ids = NotificationService::getRankedMaxId($pid, $max, $limit);
$maxId = max($ids);
$minId = min($ids);
} else {
$res = NotificationService::getMin($pid, $min ?? $since, $limit);
$ids = NotificationService::getRankedMinId($pid, $min ?? $since, $limit);
$maxId = max($ids);
$minId = min($ids);
}
$baseUrl = config('app.url') . '/api/v1/notifications?';
@ -1470,90 +1453,37 @@ class ApiV1Controller extends Controller
public function timelinePublic(Request $request)
{
$this->validate($request,[
'page' => 'nullable|integer|max:40',
'min_id' => 'nullable|integer|min:0|max:' . PHP_INT_MAX,
'max_id' => 'nullable|integer|min:0|max:' . PHP_INT_MAX,
'limit' => 'nullable|integer|max:80'
]);
$page = $request->input('page');
$min = $request->input('min_id');
$max = $request->input('max_id');
$limit = $request->input('limit') ?? 3;
$user = $request->user();
if($user) {
$key = 'user:last_active_at:id:'.$user->id;
$ttl = now()->addMinutes(5);
Cache::remember($key, $ttl, function() use($user) {
$user->last_active_at = now();
$user->save();
return;
});
}
if(PublicTimelineService::count() == 0) {
PublicTimelineService::warmCache(true, 400);
}
if($min || $max) {
$dir = $min ? '>' : '<';
$id = $min ?? $max;
$timeline = Status::select(
'id',
'uri',
'caption',
'rendered',
'profile_id',
'type',
'in_reply_to_id',
'reblog_of_id',
'is_nsfw',
'scope',
'local',
'reply_count',
'comments_disabled',
'place_id',
'likes_count',
'reblogs_count',
'created_at',
'updated_at'
)->whereNull('uri')
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album'])
->with('profile', 'hashtags', 'mentions')
->where('id', $dir, $id)
->whereScope('public')
->where('created_at', '>', now()->subDays(14))
->latest()
->limit($limit)
->get();
if ($max) {
$feed = PublicTimelineService::getRankedMaxId($max, $limit);
} else if ($min) {
$feed = PublicTimelineService::getRankedMinId($min, $limit);
} else {
$timeline = Status::select(
'id',
'uri',
'caption',
'rendered',
'profile_id',
'type',
'in_reply_to_id',
'reblog_of_id',
'is_nsfw',
'scope',
'local',
'reply_count',
'comments_disabled',
'place_id',
'likes_count',
'reblogs_count',
'created_at',
'updated_at'
)->whereNull('uri')
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album'])
->with('profile', 'hashtags', 'mentions')
->whereScope('public')
->where('created_at', '>', now()->subDays(14))
->latest()
->simplePaginate($limit);
$feed = PublicTimelineService::get(0, $limit);
}
$fractal = new Fractal\Resource\Collection($timeline, new StatusTransformer());
$res = $this->fractal->createData($fractal)->toArray();
$res = collect($feed)
->map(function($k) use($user) {
$status = StatusService::get($k);
if($user) {
$status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
}
return $status;
})
->toArray();
return response()->json($res);
}

61
app/Services/NotificationService.php
View File

@ -46,6 +46,60 @@ class NotificationService {
return $ids;
}
public static function getMax($id = false, $start, $limit = 10)
{
$ids = self::getRankedMaxId($id, $start, $limit);
if(empty($ids)) {
return [];
}
$res = collect([]);
foreach($ids as $id) {
$res->push(self::getNotification($id));
}
return $res->toArray();
}
public static function getMin($id = false, $start, $limit = 10)
{
$ids = self::getRankedMinId($id, $start, $limit);
if(empty($ids)) {
return [];
}
$res = collect([]);
foreach($ids as $id) {
$res->push(self::getNotification($id));
}
return $res->toArray();
}
public static function getRankedMaxId($id = false, $start = null, $limit = 10)
{
if(!$start || !$id) {
return [];
}
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, $start, '-inf', [
'withscores' => true,
'limit' => [1, $limit]
]));
}
public static function getRankedMinId($id = false, $end = null, $limit = 10)
{
if(!$end || !$id) {
return [];
}
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, '+inf', $end, [
'withscores' => true,
'limit' => [0, $limit]
]));
}
public static function set($id, $val)
{
return Redis::zadd(self::CACHE_KEY . $id, $val, $val);
@ -53,6 +107,7 @@ class NotificationService {
public static function del($id, $val)
{
Cache::forget('service:notification:' . $val);
return Redis::zrem(self::CACHE_KEY . $id, $val);
}
@ -73,7 +128,7 @@ class NotificationService {
public static function getNotification($id)
{
return Cache::remember('service:notification:'.$id, now()->addMonths(3), function() use($id) {
return Cache::remember('service:notification:'.$id, now()->addDays(3), function() use($id) {
$n = Notification::with('item')->findOrFail($id);
$fractal = new Fractal\Manager();
$fractal->setSerializer(new ArraySerializer());
@ -84,7 +139,7 @@ class NotificationService {
public static function setNotification(Notification $notification)
{
return Cache::remember('service:notification:'.$notification->id, now()->addMonths(3), function() use($notification) {
return Cache::remember('service:notification:'.$notification->id, now()->addDays(3), function() use($notification) {
$fractal = new Fractal\Manager();
$fractal->setSerializer(new ArraySerializer());
$resource = new Fractal\Resource\Item($notification, new NotificationTransformer());
@ -106,4 +161,4 @@ class NotificationService {
}
return 0;
}
}
}

41
app/Services/PublicTimelineService.php
View File

@ -18,18 +18,41 @@ class PublicTimelineService {
if($stop > 100) {
$stop = 100;
}
$tl = [];
$keys = Redis::zrevrange(self::CACHE_KEY, $start, $stop);
foreach($keys as $key) {
array_push($tl, StatusService::get($key));
return Redis::zrevrange(self::CACHE_KEY, $start, $stop);
}
public static function getRankedMaxId($start = null, $limit = 10)
{
if(!$start) {
return [];
}
return $tl;
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, $start, '-inf', [
'withscores' => true,
'limit' => [1, $limit]
]));
}
public static function getRankedMinId($end = null, $limit = 10)
{
if(!$end) {
return [];
}
return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, '+inf', $end, [
'withscores' => true,
'limit' => [0, $limit]
]));
}
public static function add($val)
{
// return Redis::zadd(self::CACHE_KEY, $val, $val);
return;
if(self::count() > 400) {
Redis::zpopmin(self::CACHE_KEY);
}
return Redis::zadd(self::CACHE_KEY, $val, $val);
}
public static function rem($val)
@ -44,7 +67,7 @@ class PublicTimelineService {
public static function count()
{
return Redis::zcount(self::CACHE_KEY, '-inf', '+inf');
return Redis::zcard(self::CACHE_KEY);
}
public static function warmCache($force = false, $limit = 100)
@ -55,7 +78,7 @@ class PublicTimelineService {
->whereNull('reblog_of_id')
->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
->whereScope('public')
->latest()
->orderByDesc('id')
->limit($limit)
->pluck('id');
foreach($ids as $id) {

3
app/Services/StatusService.php
View File

@ -37,6 +37,7 @@ class StatusService {
public static function del($id)
{
PublicTimelineService::rem($id);
return Cache::forget(self::key($id));
}
}
}

1
resources/views/admin/instances/home.blade.php
View File

@ -85,6 +85,7 @@
@endsection
@push('scripts')
<script type="text/javascript" src="{{mix('js/components.js')}}"></script>
<script type="text/javascript">
$(document).ready(function() {
$('.filesize').each(function(k,v) {

3
resources/views/admin/instances/show.blade.php
View File

@ -103,6 +103,7 @@
@endsection
@push('scripts')
<script type="text/javascript" src="{{mix('js/components.js')}}"></script>
<script type="text/javascript">
$(document).ready(function() {
$('.filesize').each(function(k,v) {
@ -110,4 +111,4 @@
});
});
</script>
@endpush
@endpush