chris/pixelfed
1
0
Fork 0
forked from mirror/pixelfed
Code Pull requests Activity

Update HomeFeedPipeline, fix tag filtering

Browse source
This commit is contained in:
Daniel Supernault 2023-11-16 00:47:49 -07:00
parent e5401f8558
commit f105f4e8f6
No known key found for this signature in database
GPG key ID: 23740873EE6F76A1
6 changed files with 267 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.editorconfig
View file

@ -1,8 +1,8 @@
root = true
[*]
indent_style = space
indent_size = 4
indent_style = tab
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true

13
app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php
View file

@ -12,6 +12,7 @@ use App\Hashtag;
use App\StatusHashtag;
use App\Services\HashtagFollowService;
use App\Services\HomeTimelineService;
use App\Services\StatusService;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
@ -72,11 +73,21 @@ class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
public function handle(): void
{
$hashtag = $this->hashtag;
$sid = $hashtag->status_id;
$status = StatusService::get($sid, false);
if(!$status) {
return;
}
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
return;
}
$ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id);
if(!$ids || !count($ids)) {
return;
return;
}
foreach($ids as $id) {

10
app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php
View file

@ -12,6 +12,7 @@ use App\Hashtag;
use App\StatusHashtag;
use App\Services\HashtagFollowService;
use App\Services\HomeTimelineService;
use App\Services\StatusService;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
@ -68,6 +69,15 @@ class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
{
$sid = $this->sid;
$hid = $this->hid;
$status = StatusService::get($sid, false);
if(!$status) {
return;
}
if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
return;
}
$ids = HashtagFollowService::getPidByHid($hid);

10
app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php
View file

@ -48,7 +48,7 @@ class HashtagUnfollowPipeline implements ShouldQueue
{
$hid = $this->hid;
$pid = $this->pid;
$slug = $this->slug;
$slug = strtolower($this->slug);
$statusIds = HomeTimelineService::get($pid, 0, -1);
@ -59,17 +59,17 @@ class HashtagUnfollowPipeline implements ShouldQueue
foreach($statusIds as $id) {
$status = StatusService::get($id, false);
if(!$status) {
if(!$status || empty($status['tags'])) {
HomeTimelineService::rem($pid, $id);
continue;
}
$following = in_array($status['account']['id'], $followingIds);
if($following || !isset($status['tags'])) {
$following = in_array((int) $status['account']['id'], $followingIds);
if($following === true) {
continue;
}
$tags = collect($status['tags'])->map(function($tag) {
return $tag['name'];
return strtolower($tag['name']);
})->filter()->values()->toArray();
if(in_array($slug, $tags)) {

290
app/Jobs/StatusPipeline/StatusEntityLexer.php
View file

@ -19,6 +19,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use App\Services\StatusService;
use App\Services\UserFilterService;
use App\Services\AdminShadowFilterService;
use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
@ -26,87 +27,87 @@ use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
class StatusEntityLexer implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $status;
protected $entities;
protected $autolink;
protected $status;
protected $entities;
protected $autolink;
/**
* Delete the job if its models no longer exist.
*
* @var bool
*/
public $deleteWhenMissingModels = true;
/**
* Delete the job if its models no longer exist.
*
* @var bool
*/
public $deleteWhenMissingModels = true;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(Status $status)
{
$this->status = $status;
}
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(Status $status)
{
$this->status = $status;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$profile = $this->status->profile;
$status = $this->status;
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$profile = $this->status->profile;
$status = $this->status;
if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
$profile->status_count = $profile->status_count + 1;
$profile->save();
}
if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
$profile->status_count = $profile->status_count + 1;
$profile->save();
}
if($profile->no_autolink == false) {
$this->parseEntities();
}
}
if($profile->no_autolink == false) {
$this->parseEntities();
}
}
public function parseEntities()
{
$this->extractEntities();
}
public function parseEntities()
{
$this->extractEntities();
}
public function extractEntities()
{
$this->entities = Extractor::create()->extract($this->status->caption);
$this->autolinkStatus();
}
public function extractEntities()
{
$this->entities = Extractor::create()->extract($this->status->caption);
$this->autolinkStatus();
}
public function autolinkStatus()
{
$this->autolink = Autolink::create()->autolink($this->status->caption);
$this->storeEntities();
}
public function autolinkStatus()
{
$this->autolink = Autolink::create()->autolink($this->status->caption);
$this->storeEntities();
}
public function storeEntities()
{
$this->storeHashtags();
DB::transaction(function () {
$status = $this->status;
$status->rendered = nl2br($this->autolink);
$status->save();
});
}
public function storeEntities()
{
$this->storeHashtags();
DB::transaction(function () {
$status = $this->status;
$status->rendered = nl2br($this->autolink);
$status->save();
});
}
public function storeHashtags()
{
$tags = array_unique($this->entities['hashtags']);
$status = $this->status;
public function storeHashtags()
{
$tags = array_unique($this->entities['hashtags']);
$status = $this->status;
foreach ($tags as $tag) {
if(mb_strlen($tag) > 124) {
continue;
}
DB::transaction(function () use ($status, $tag) {
$slug = str_slug($tag, '-', false);
foreach ($tags as $tag) {
if(mb_strlen($tag) > 124) {
continue;
}
DB::transaction(function () use ($status, $tag) {
$slug = str_slug($tag, '-', false);
$hashtag = Hashtag::firstOrCreate([
'slug' => $slug
@ -114,92 +115,93 @@ class StatusEntityLexer implements ShouldQueue
'name' => $tag
]);
StatusHashtag::firstOrCreate(
[
'status_id' => $status->id,
'hashtag_id' => $hashtag->id,
'profile_id' => $status->profile_id,
'status_visibility' => $status->visibility,
]
);
});
}
$this->storeMentions();
}
StatusHashtag::firstOrCreate(
[
'status_id' => $status->id,
'hashtag_id' => $hashtag->id,
'profile_id' => $status->profile_id,
'status_visibility' => $status->visibility,
]
);
});
}
$this->storeMentions();
}
public function storeMentions()
{
$mentions = array_unique($this->entities['mentions']);
$status = $this->status;
public function storeMentions()
{
$mentions = array_unique($this->entities['mentions']);
$status = $this->status;
foreach ($mentions as $mention) {
$mentioned = Profile::whereUsername($mention)->first();
foreach ($mentions as $mention) {
$mentioned = Profile::whereUsername($mention)->first();
if (empty($mentioned) || !isset($mentioned->id)) {
continue;
}
if (empty($mentioned) || !isset($mentioned->id)) {
continue;
}
$blocks = UserFilterService::blocks($mentioned->id);
if($blocks && in_array($status->profile_id, $blocks)) {
continue;
}
DB::transaction(function () use ($status, $mentioned) {
$m = new Mention();
$m->status_id = $status->id;
$m->profile_id = $mentioned->id;
$m->save();
DB::transaction(function () use ($status, $mentioned) {
$m = new Mention();
$m->status_id = $status->id;
$m->profile_id = $mentioned->id;
$m->save();
MentionPipeline::dispatch($status, $m);
});
}
$this->fanout();
}
MentionPipeline::dispatch($status, $m);
});
}
$this->fanout();
}
public function fanout()
{
$status = $this->status;
public function fanout()
{
$status = $this->status;
StatusService::refresh($status->id);
if(config('exp.cached_home_timeline')) {
if( $status->in_reply_to_id === null &&
in_array($status->scope, ['public', 'unlisted', 'private'])
) {
FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
}
}
$this->deliver();
}
public function deliver()
{
$status = $this->status;
$types = [
'photo',
'photo:album',
'video',
'video:album',
'photo:video:album'
];
if(config_cache('pixelfed.bouncer.enabled')) {
Bouncer::get($status);
}
Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
if( $status->uri == null &&
$status->scope == 'public' &&
in_array($status->type, $types) &&
$status->in_reply_to_id === null &&
$status->reblog_of_id === null &&
($hideNsfw ? $status->is_nsfw == false : true)
) {
if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
PublicTimelineService::add($status->id);
if(config('exp.cached_home_timeline')) {
if( $status->in_reply_to_id === null &&
in_array($status->scope, ['public', 'unlisted', 'private'])
) {
FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
}
}
}
$this->deliver();
}
if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
StatusActivityPubDeliver::dispatch($status);
}
}
public function deliver()
{
$status = $this->status;
$types = [
'photo',
'photo:album',
'video',
'video:album',
'photo:video:album'
];
if(config_cache('pixelfed.bouncer.enabled')) {
Bouncer::get($status);
}
Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
if( $status->uri == null &&
$status->scope == 'public' &&
in_array($status->type, $types) &&
$status->in_reply_to_id === null &&
$status->reblog_of_id === null &&
($hideNsfw ? $status->is_nsfw == false : true)
) {
if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
PublicTimelineService::add($status->id);
}
}
if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
StatusActivityPubDeliver::dispatch($status);
}
}
}

184
app/Jobs/StatusPipeline/StatusTagsPipeline.php
View file

@ -20,117 +20,119 @@ use App\Util\ActivityPub\Helpers;
class StatusTagsPipeline implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
protected $activity;
protected $status;
protected $activity;
protected $status;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($activity, $status)
{
$this->activity = $activity;
$this->status = $status;
}
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($activity, $status)
{
$this->activity = $activity;
$this->status = $status;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$res = $this->activity;
$status = $this->status;
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$res = $this->activity;
$status = $this->status;
if(isset($res['tag']['type'], $res['tag']['name'])) {
$res['tag'] = [$res['tag']];
}
$tags = collect($res['tag']);
$tags = collect($res['tag']);
// Emoji
$tags->filter(function($tag) {
return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
})
->map(function($tag) {
CustomEmojiService::import($tag['id'], $this->status->id);
});
// Emoji
$tags->filter(function($tag) {
return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
})
->map(function($tag) {
CustomEmojiService::import($tag['id'], $this->status->id);
});
// Hashtags
$tags->filter(function($tag) {
return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
})
->map(function($tag) use($status) {
$name = substr($tag['name'], 0, 1) == '#' ?
substr($tag['name'], 1) : $tag['name'];
// Hashtags
$tags->filter(function($tag) {
return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
})
->map(function($tag) use($status) {
$name = substr($tag['name'], 0, 1) == '#' ?
substr($tag['name'], 1) : $tag['name'];
$banned = TrendingHashtagService::getBannedHashtagNames();
$banned = TrendingHashtagService::getBannedHashtagNames();
if(count($banned)) {
if(count($banned)) {
if(in_array(strtolower($name), array_map('strtolower', $banned))) {
return;
return;
}
}
if(config('database.default') === 'pgsql') {
$hashtag = Hashtag::where('name', 'ilike', $name)
->orWhere('slug', 'ilike', str_slug($name, '-', false))
->first();
$hashtag = Hashtag::where('name', 'ilike', $name)
->orWhere('slug', 'ilike', str_slug($name, '-', false))
->first();
if(!$hashtag) {
$hashtag = Hashtag::updateOrCreate([
'slug' => str_slug($name, '-', false),
'name' => $name
]);
}
if(!$hashtag) {
$hashtag = Hashtag::updateOrCreate([
'slug' => str_slug($name, '-', false),
'name' => $name
]);
}
} else {
$hashtag = Hashtag::updateOrCreate([
'slug' => str_slug($name, '-', false),
'name' => $name
]);
$hashtag = Hashtag::updateOrCreate([
'slug' => str_slug($name, '-', false),
'name' => $name
]);
}
StatusHashtag::firstOrCreate([
'status_id' => $status->id,
'hashtag_id' => $hashtag->id,
'profile_id' => $status->profile_id,
'status_visibility' => $status->scope
]);
});
StatusHashtag::firstOrCreate([
'status_id' => $status->id,
'hashtag_id' => $hashtag->id,
'profile_id' => $status->profile_id,
'status_visibility' => $status->scope
]);
});
// Mentions
$tags->filter(function($tag) {
return $tag &&
$tag['type'] == 'Mention' &&
isset($tag['href']) &&
substr($tag['href'], 0, 8) === 'https://';
})
->map(function($tag) use($status) {
if(Helpers::validateLocalUrl($tag['href'])) {
$parts = explode('/', $tag['href']);
if(!$parts) {
return;
}
$pid = AccountService::usernameToId(end($parts));
if(!$pid) {
return;
}
} else {
$acct = Helpers::profileFetch($tag['href']);
if(!$acct) {
return;
}
$pid = $acct->id;
}
$mention = new Mention;
$mention->status_id = $status->id;
$mention->profile_id = $pid;
$mention->save();
MentionPipeline::dispatch($status, $mention);
});
}
// Mentions
$tags->filter(function($tag) {
return $tag &&
$tag['type'] == 'Mention' &&
isset($tag['href']) &&
substr($tag['href'], 0, 8) === 'https://';
})
->map(function($tag) use($status) {
if(Helpers::validateLocalUrl($tag['href'])) {
$parts = explode('/', $tag['href']);
if(!$parts) {
return;
}
$pid = AccountService::usernameToId(end($parts));
if(!$pid) {
return;
}
} else {
$acct = Helpers::profileFetch($tag['href']);
if(!$acct) {
return;
}
$pid = $acct->id;
}
$mention = new Mention;
$mention->status_id = $status->id;
$mention->profile_id = $pid;
$mention->save();
MentionPipeline::dispatch($status, $mention);
});
StatusService::refresh($status->id);
}
}