Merge pull request #4777 from pixelfed/staging

Add Related Hashtags
This commit is contained in:
daniel 2023-11-17 22:25:06 -07:00 committed by GitHub
commit 54b6c96112
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 3762 additions and 3538 deletions

View File

@ -10,6 +10,7 @@
- Added S3 command to rewrite media urls ([5b3a5610](https://github.com/pixelfed/pixelfed/commit/5b3a5610))
- Experimental home feed ([#4752](https://github.com/pixelfed/pixelfed/pull/4752)) ([c39b9afb](https://github.com/pixelfed/pixelfed/commit/c39b9afb))
- Added `app:hashtag-cached-count-update` command to update cached_count of hashtags and add to scheduler to run every 25 minutes past the hour ([1e31fee6](https://github.com/pixelfed/pixelfed/commit/1e31fee6))
- Added `app:hashtag-related-generate` command to generate related hashtags ([176b4ed7](https://github.com/pixelfed/pixelfed/commit/176b4ed7))
### Federation
- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e))
@ -58,6 +59,8 @@
- Update StatusHashtagService, remove problematic cache layer ([e5401f85](https://github.com/pixelfed/pixelfed/commit/e5401f85))
- Update HomeFeedPipeline, fix tag filtering ([f105f4e8](https://github.com/pixelfed/pixelfed/commit/f105f4e8))
- Update HashtagService, reduce cached_count cache ttl ([15f29f7d](https://github.com/pixelfed/pixelfed/commit/15f29f7d))
- Update ApiV1Controller, fix include_reblogs param on timelines/home endpoint, and improve limit pagination logic ([287f903b](https://github.com/pixelfed/pixelfed/commit/287f903b))
- ([](https://github.com/pixelfed/pixelfed/commit/))
- ([](https://github.com/pixelfed/pixelfed/commit/))
## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9)

View File

@ -0,0 +1,83 @@
<?php
namespace App\Console\Commands;
use Illuminate\Console\Command;
use App\Hashtag;
use App\StatusHashtag;
use App\Models\HashtagRelated;
use App\Services\HashtagRelatedService;
use Illuminate\Contracts\Console\PromptsForMissingInput;
use function Laravel\Prompts\multiselect;
class HashtagRelatedGenerate extends Command implements PromptsForMissingInput
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'app:hashtag-related-generate {tag}';
/**
* The console command description.
*
* @var string
*/
protected $description = 'Command description';
/**
* Prompt for missing input arguments using the returned questions.
*
* @return array
*/
protected function promptForMissingArgumentsUsing()
{
return [
'tag' => 'Which hashtag should we generate related tags for?',
];
}
/**
* Execute the console command.
*/
public function handle()
{
$tag = $this->argument('tag');
$hashtag = Hashtag::whereName($tag)->orWhere('slug', $tag)->first();
if(!$hashtag) {
$this->error('Hashtag not found, aborting...');
exit;
}
$this->info('Looking up #' . $tag . '...');
$tags = StatusHashtag::whereHashtagId($hashtag->id)->count();
if(!$tags || $tags < 100) {
$this->error('Not enough posts found to generate related hashtags!');
exit;
}
$this->info('Found ' . $tags . ' posts that use that hashtag');
$related = collect(HashtagRelatedService::fetchRelatedTags($tag));
$selected = multiselect(
label: 'Which tags do you want to generate?',
options: $related->pluck('name'),
required: true,
);
$filtered = $related->filter(fn($i) => in_array($i['name'], $selected))->all();
$agg_score = $related->filter(fn($i) => in_array($i['name'], $selected))->sum('related_count');
HashtagRelated::updateOrCreate([
'hashtag_id' => $hashtag->id,
], [
'related_tags' => array_values($filtered),
'agg_score' => $agg_score,
'last_calculated_at' => now()
]);
$this->info('Finished!');
}
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,207 @@
<?php
namespace App\Http\Controllers\Api\V1;
use Illuminate\Http\Request;
use App\Http\Controllers\Controller;
use App\Hashtag;
use App\HashtagFollow;
use App\StatusHashtag;
use App\Services\AccountService;
use App\Services\HashtagService;
use App\Services\HashtagFollowService;
use App\Services\HashtagRelatedService;
use App\Http\Resources\MastoApi\FollowedTagResource;
use App\Jobs\HomeFeedPipeline\FeedWarmCachePipeline;
use App\Jobs\HomeFeedPipeline\HashtagUnfollowPipeline;
class TagsController extends Controller
{
const PF_API_ENTITY_KEY = "_pe";
public function json($res, $code = 200, $headers = [])
{
return response()->json($res, $code, $headers, JSON_UNESCAPED_SLASHES);
}
/**
* GET /api/v1/tags/:id/related
*
*
* @return array
*/
public function relatedTags(Request $request, $tag)
{
abort_unless($request->user(), 403);
$tag = Hashtag::whereSlug($tag)->firstOrFail();
return HashtagRelatedService::get($tag->id);
}
/**
* POST /api/v1/tags/:id/follow
*
*
* @return object
*/
public function followHashtag(Request $request, $id)
{
abort_if(!$request->user(), 403);
$pid = $request->user()->profile_id;
$account = AccountService::get($pid);
$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
$tag = Hashtag::where('name', $operator, $id)
->orWhere('slug', $operator, $id)
->first();
abort_if(!$tag, 422, 'Unknown hashtag');
abort_if(
HashtagFollow::whereProfileId($pid)->count() >= HashtagFollow::MAX_LIMIT,
422,
'You cannot follow more than ' . HashtagFollow::MAX_LIMIT . ' hashtags.'
);
$follows = HashtagFollow::updateOrCreate(
[
'profile_id' => $account['id'],
'hashtag_id' => $tag->id
],
[
'user_id' => $request->user()->id
]
);
HashtagService::follow($pid, $tag->id);
HashtagFollowService::add($tag->id, $pid);
return response()->json(FollowedTagResource::make($follows)->toArray($request));
}
/**
* POST /api/v1/tags/:id/unfollow
*
*
* @return object
*/
public function unfollowHashtag(Request $request, $id)
{
abort_if(!$request->user(), 403);
$pid = $request->user()->profile_id;
$account = AccountService::get($pid);
$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
$tag = Hashtag::where('name', $operator, $id)
->orWhere('slug', $operator, $id)
->first();
abort_if(!$tag, 422, 'Unknown hashtag');
$follows = HashtagFollow::whereProfileId($pid)
->whereHashtagId($tag->id)
->first();
if(!$follows) {
return [
'name' => $tag->name,
'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
'history' => [],
'following' => false
];
}
if($follows) {
HashtagService::unfollow($pid, $tag->id);
HashtagFollowService::unfollow($tag->id, $pid);
HashtagUnfollowPipeline::dispatch($tag->id, $pid, $tag->slug)->onQueue('feed');
$follows->delete();
}
$res = FollowedTagResource::make($follows)->toArray($request);
$res['following'] = false;
return response()->json($res);
}
/**
* GET /api/v1/tags/:id
*
*
* @return object
*/
public function getHashtag(Request $request, $id)
{
abort_if(!$request->user(), 403);
$pid = $request->user()->profile_id;
$account = AccountService::get($pid);
$operator = config('database.default') == 'pgsql' ? 'ilike' : 'like';
$tag = Hashtag::where('name', $operator, $id)
->orWhere('slug', $operator, $id)
->first();
if(!$tag) {
return [
'name' => $id,
'url' => config('app.url') . '/i/web/hashtag/' . $id,
'history' => [],
'following' => false
];
}
$res = [
'name' => $tag->name,
'url' => config('app.url') . '/i/web/hashtag/' . $tag->slug,
'history' => [],
'following' => HashtagService::isFollowing($pid, $tag->id)
];
if($request->has(self::PF_API_ENTITY_KEY)) {
$res['count'] = HashtagService::count($tag->id);
}
return $this->json($res);
}
/**
* GET /api/v1/followed_tags
*
*
* @return array
*/
public function getFollowedTags(Request $request)
{
abort_if(!$request->user(), 403);
$account = AccountService::get($request->user()->profile_id);
$this->validate($request, [
'cursor' => 'sometimes',
'limit' => 'sometimes|integer|min:1|max:200'
]);
$limit = $request->input('limit', 100);
$res = HashtagFollow::whereProfileId($account['id'])
->orderByDesc('id')
->cursorPaginate($limit)
->withQueryString();
$pagination = false;
$prevPage = $res->nextPageUrl();
$nextPage = $res->previousPageUrl();
if($nextPage && $prevPage) {
$pagination = '<' . $nextPage . '>; rel="next", <' . $prevPage . '>; rel="prev"';
} else if($nextPage && !$prevPage) {
$pagination = '<' . $nextPage . '>; rel="next"';
} else if(!$nextPage && $prevPage) {
$pagination = '<' . $prevPage . '>; rel="prev"';
}
if($pagination) {
return response()->json(FollowedTagResource::collection($res)->collection)
->header('Link', $pagination);
}
return response()->json(FollowedTagResource::collection($res)->collection);
}
}

View File

@ -0,0 +1,24 @@
<?php
namespace App\Models;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
class HashtagRelated extends Model
{
use HasFactory;
protected $guarded = [];
/**
* The attributes that should be mutated to dates and other custom formats.
*
* @var array
*/
protected $casts = [
'related_tags' => 'array',
'last_calculated_at' => 'datetime',
'last_moderated_at' => 'datetime',
];
}

View File

@ -0,0 +1,38 @@
<?php
namespace App\Services;
use DB;
use App\StatusHashtag;
use App\Models\HashtagRelated;
class HashtagRelatedService
{
public static function get($id)
{
$tag = HashtagRelated::whereHashtagId($id)->first();
if(!$tag) {
return [];
}
return $tag->related_tags;
}
public static function fetchRelatedTags($tag)
{
$res = StatusHashtag::query()
->select('h2.name', DB::raw('COUNT(*) as related_count'))
->join('status_hashtags as hs2', function ($join) {
$join->on('status_hashtags.status_id', '=', 'hs2.status_id')
->whereRaw('status_hashtags.hashtag_id != hs2.hashtag_id');
})
->join('hashtags as h1', 'status_hashtags.hashtag_id', '=', 'h1.id')
->join('hashtags as h2', 'hs2.hashtag_id', '=', 'h2.id')
->where('h1.name', '=', $tag)
->groupBy('h2.name')
->orderBy('related_count', 'desc')
->limit(30)
->get();
return $res;
}
}

View File

@ -0,0 +1,33 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::create('hashtag_related', function (Blueprint $table) {
$table->bigIncrements('id');
$table->bigInteger('hashtag_id')->unsigned()->unique()->index();
$table->json('related_tags')->nullable();
$table->bigInteger('agg_score')->unsigned()->nullable()->index();
$table->timestamp('last_calculated_at')->nullable()->index();
$table->timestamp('last_moderated_at')->nullable()->index();
$table->boolean('skip_refresh')->default(false)->index();
$table->timestamps();
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::dropIfExists('hashtag_related');
}
};

View File

@ -2,6 +2,7 @@
use Illuminate\Http\Request;
use App\Http\Middleware\DeprecatedEndpoint;
use App\Http\Controllers\Api\V1\TagsController;
$middleware = ['auth:api','validemail'];
@ -92,10 +93,11 @@ Route::group(['prefix' => 'api'], function() use($middleware) {
Route::get('markers', 'Api\ApiV1Controller@getMarkers')->middleware($middleware);
Route::post('markers', 'Api\ApiV1Controller@setMarkers')->middleware($middleware);
Route::get('followed_tags', 'Api\ApiV1Controller@getFollowedTags')->middleware($middleware);
Route::post('tags/{id}/follow', 'Api\ApiV1Controller@followHashtag')->middleware($middleware);
Route::post('tags/{id}/unfollow', 'Api\ApiV1Controller@unfollowHashtag')->middleware($middleware);
Route::get('tags/{id}', 'Api\ApiV1Controller@getHashtag')->middleware($middleware);
Route::get('followed_tags', [TagsController::class, 'getFollowedTags'])->middleware($middleware);
Route::post('tags/{id}/follow', [TagsController::class, 'followHashtag'])->middleware($middleware);
Route::post('tags/{id}/unfollow', [TagsController::class, 'unfollowHashtag'])->middleware($middleware);
Route::get('tags/{id}/related', [TagsController::class, 'relatedTags'])->middleware($middleware);
Route::get('tags/{id}', [TagsController::class, 'getHashtag'])->middleware($middleware);
Route::get('statuses/{id}/history', 'StatusEditController@history')->middleware($middleware);
Route::put('statuses/{id}', 'StatusEditController@store')->middleware($middleware);