diff --git a/CHANGELOG.md b/CHANGELOG.md index bdfa38426..a8a2c1be7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,41 @@ ## [Unreleased](https://github.com/pixelfed/pixelfed/compare/v0.11.9...dev) -## [v0.11.9 (2023-08-06)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) +### Added +- Resilient Media Storage ([#4665](https://github.com/pixelfed/pixelfed/pull/4665)) ([fb1deb6](https://github.com/pixelfed/pixelfed/commit/fb1deb6)) +- Added user:2fa command to easily disable 2FA for given account ([c6408fd7](https://github.com/pixelfed/pixelfed/commit/c6408fd7)) +- Added `avatar:storage-deep-clean` command to dispatch remote avatar storage cleanup jobs ([c37b7cde](https://github.com/pixelfed/pixelfed/commit/c37b7cde)) + +### Federation +- Update Privacy Settings, add support for Mastodon `indexable` search flag ([fc24630e](https://github.com/pixelfed/pixelfed/commit/fc24630e)) +- Update AP Helpers, consume actor `indexable` attribute ([fbdcdd9d](https://github.com/pixelfed/pixelfed/commit/fbdcdd9d)) +- ([](https://github.com/pixelfed/pixelfed/commit/)) + +### Updates +- Update FollowerService, add forget method to RelationshipService call to reduce load when mass purging ([347e4f59](https://github.com/pixelfed/pixelfed/commit/347e4f59)) +- Update FollowServiceWarmCache, improve handling larger following/follower lists ([61a6d904](https://github.com/pixelfed/pixelfed/commit/61a6d904)) +- Update StoryApiV1Controller, add viewers route to view story viewers ([941736ce](https://github.com/pixelfed/pixelfed/commit/941736ce)) +- Update NotificationService, improve cache warming query ([2496386d](https://github.com/pixelfed/pixelfed/commit/2496386d)) +- Update StatusService, hydrate accounts on request instead of caching them along with status objects ([223661ec](https://github.com/pixelfed/pixelfed/commit/223661ec)) +- Update profile embed, fix resize ([dc23c21d](https://github.com/pixelfed/pixelfed/commit/dc23c21d)) +- Update Status model, improve thumb logic ([d969a973](https://github.com/pixelfed/pixelfed/commit/d969a973)) +- Update Status model, allow unlisted thumbnails ([1f0a45b7](https://github.com/pixelfed/pixelfed/commit/1f0a45b7)) +- Update StatusTagsPipeline, fix object tags and slug normalization ([d295e605](https://github.com/pixelfed/pixelfed/commit/d295e605)) +- Update Note and CreateNote transformers, include attachment blurhash, width and height ([ce1afe27](https://github.com/pixelfed/pixelfed/commit/ce1afe27)) +- Update ap helpers, store media attachment width and height if present ([8c969191](https://github.com/pixelfed/pixelfed/commit/8c969191)) +- Update Sign-in with Mastodon, allow usage when registrations are closed ([895dc4fa](https://github.com/pixelfed/pixelfed/commit/895dc4fa)) +- Update profile embeds, filter sensitive posts ([ede5ec3b](https://github.com/pixelfed/pixelfed/commit/ede5ec3b)) +- Update ApiV1Controller, hydrate reblog interactions. Fixes ([#4686](https://github.com/pixelfed/pixelfed/issues/4686)) ([135798eb](https://github.com/pixelfed/pixelfed/commit/135798eb)) +- Update AdminReportController, add `profile_id` to group by. Fixes ([#4685](https://github.com/pixelfed/pixelfed/issues/4685)) ([e4d3b196](https://github.com/pixelfed/pixelfed/commit/e4d3b196)) +- Update user:admin command, improve logic. Fixes ([#2465](https://github.com/pixelfed/pixelfed/issues/2465)) ([01bac511](https://github.com/pixelfed/pixelfed/commit/01bac511)) +- Update AP helpers, adjust RemoteAvatarFetch ttl from 24h to 3 months ([36b23fe3](https://github.com/pixelfed/pixelfed/commit/36b23fe3)) +- Update AvatarPipeline, improve refresh logic and garbage collection to purge old avatars ([82798b5e](https://github.com/pixelfed/pixelfed/commit/82798b5e)) +- Update CreateAvatar job, add processing constraints and set `is_remote` attribute ([319ced40](https://github.com/pixelfed/pixelfed/commit/319ced40)) +- Update RemoteStatusDelete and DecrementPostCount pipelines ([edbcf3ed](https://github.com/pixelfed/pixelfed/commit/edbcf3ed)) +- Update lexer regex, fix mention regex and add more tests ([778e83d3](https://github.com/pixelfed/pixelfed/commit/778e83d3)) +- ([](https://github.com/pixelfed/pixelfed/commit/)) + +## [v0.11.9 (2023-08-21)](https://github.com/pixelfed/pixelfed/compare/v0.11.8...v0.11.9) ### Added - Import from Instagram ([#4466](https://github.com/pixelfed/pixelfed/pull/4466)) ([cf3078c5](https://github.com/pixelfed/pixelfed/commit/cf3078c5)) @@ -57,8 +91,7 @@ - Update RemoteStatusDelete pipeline ([71e92261](https://github.com/pixelfed/pixelfed/commit/71e92261)) - Update RemoteStatusDelete pipeline ([fab8f25e](https://github.com/pixelfed/pixelfed/commit/fab8f25e)) - Update RemoteStatusPipeline, fix reply check ([618b6727](https://github.com/pixelfed/pixelfed/commit/618b6727)) -- ([](https://github.com/pixelfed/pixelfed/commit/)) -- ([](https://github.com/pixelfed/pixelfed/commit/)) +- Update ApiV1Controller, add bookmarked to timeline entities ([ca746717](https://github.com/pixelfed/pixelfed/commit/ca746717)) ## [v0.11.8 (2023-05-29)](https://github.com/pixelfed/pixelfed/compare/v0.11.7...v0.11.8) diff --git a/app/Console/Commands/AvatarStorageDeepClean.php b/app/Console/Commands/AvatarStorageDeepClean.php new file mode 100644 index 000000000..5840142f5 --- /dev/null +++ b/app/Console/Commands/AvatarStorageDeepClean.php @@ -0,0 +1,115 @@ +info(' ____ _ ______ __ '); + $this->info(' / __ \(_) _____ / / __/__ ____/ / '); + $this->info(' / /_/ / / |/_/ _ \/ / /_/ _ \/ __ / '); + $this->info(' / ____/ /> info(' /_/ /_/_/|_|\___/_/_/ \___/\__,_/ '); + $this->info(' '); + $this->info(' Pixelfed Avatar Deep Cleaner'); + $this->line(' '); + $this->info(' Purge/delete old and outdated avatars from remote accounts'); + $this->line(' '); + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + $this->error('Remote avatars are not cached locally, there is nothing to purge. Aborting...'); + exit; + } + + $start = 0; + + if(!$this->confirm('Are you sure you want to proceed?')) { + $this->error('Aborting...'); + exit; + } + + if(!$this->activeCheck()) { + $this->info('Found existing deep cleaning job'); + if(!$this->confirm('Do you want to continue where you left off?')) { + $this->error('Aborting...'); + exit; + } else { + $start = Cache::has('cmd:asdp') ? (int) Cache::get('cmd:asdp') : (int) Storage::get('avatar-deep-clean.json'); + + if($start && $start < 1 || $start > PHP_INT_MAX) { + $this->error('Error fetching cached value'); + $this->error('Aborting...'); + exit; + } + } + } + + $count = Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->count(); + $bar = $this->output->createProgressBar($count); + + foreach(Avatar::whereNotNull('cdn_url')->where('is_remote', true)->where('id', '>', $start)->lazyById(10, 'id') as $avatar) { + usleep(random_int(50, 1000)); + $this->counter++; + $this->handleAvatar($avatar); + $bar->advance(); + } + $bar->finish(); + } + + protected function updateCache($id) + { + Cache::put('cmd:asdp', $id); + if($this->counter % 5 === 0) { + Storage::put('avatar-deep-clean.json', $id); + } + } + + protected function activeCheck() + { + if(Storage::exists('avatar-deep-clean.json') || Cache::has('cmd:asdp')) { + return false; + } + + return true; + } + + protected function handleAvatar($avatar) + { + $this->updateCache($avatar->id); + $queues = ['feed', 'mmo', 'feed', 'mmo', 'feed', 'feed', 'mmo', 'low']; + $queue = $queues[random_int(0, 7)]; + AvatarStorageCleanup::dispatch($avatar)->onQueue($queue); + } +} diff --git a/app/Console/Commands/UserAdmin.php b/app/Console/Commands/UserAdmin.php index 8a485c52b..3b15d0dfa 100644 --- a/app/Console/Commands/UserAdmin.php +++ b/app/Console/Commands/UserAdmin.php @@ -3,16 +3,17 @@ namespace App\Console\Commands; use Illuminate\Console\Command; +use Illuminate\Contracts\Console\PromptsForMissingInput; use App\User; -class UserAdmin extends Command +class UserAdmin extends Command implements PromptsForMissingInput { /** * The name and signature of the console command. * * @var string */ - protected $signature = 'user:admin {id}'; + protected $signature = 'user:admin {username}'; /** * The console command description. @@ -22,13 +23,15 @@ class UserAdmin extends Command protected $description = 'Make a user an admin, or remove admin privileges.'; /** - * Create a new command instance. + * Prompt for missing input arguments using the returned questions. * - * @return void + * @return array */ - public function __construct() + protected function promptForMissingArgumentsUsing() { - parent::__construct(); + return [ + 'username' => 'Which username should we toggle admin privileges for?', + ]; } /** @@ -38,16 +41,15 @@ class UserAdmin extends Command */ public function handle() { - $id = $this->argument('id'); - if(ctype_digit($id) == true) { - $user = User::find($id); - } else { - $user = User::whereUsername($id)->first(); - } + $id = $this->argument('username'); + + $user = User::whereUsername($id)->first(); + if(!$user) { $this->error('Could not find any user with that username or id.'); exit; } + $this->info('Found username: ' . $user->username); $state = $user->is_admin ? 'Remove admin privileges from this user?' : 'Add admin privileges to this user?'; $confirmed = $this->confirm($state); diff --git a/app/Console/Commands/UserToggle2FA.php b/app/Console/Commands/UserToggle2FA.php new file mode 100644 index 000000000..eed6843da --- /dev/null +++ b/app/Console/Commands/UserToggle2FA.php @@ -0,0 +1,61 @@ + 'Which username should we disable 2FA for?', + ]; + } + + /** + * Execute the console command. + */ + public function handle() + { + $user = User::whereUsername($this->argument('username'))->first(); + + if(!$user) { + $this->error('Could not find any user with that username'); + exit; + } + + if(!$user->{'2fa_enabled'}) { + $this->info('User did not have 2FA enabled!'); + return; + } + + $user->{'2fa_enabled'} = false; + $user->{'2fa_secret'} = null; + $user->{'2fa_backup_codes'} = null; + $user->save(); + + $this->info('Successfully disabled 2FA on this account!'); + } +} diff --git a/app/Http/Controllers/Admin/AdminReportController.php b/app/Http/Controllers/Admin/AdminReportController.php index 4924acfa8..ac238f28c 100644 --- a/app/Http/Controllers/Admin/AdminReportController.php +++ b/app/Http/Controllers/Admin/AdminReportController.php @@ -643,7 +643,7 @@ trait AdminReportController $q->whereNull('admin_seen') : $q->whereNotNull('admin_seen'); }) - ->groupBy(['object_id', 'object_type']) + ->groupBy(['id', 'object_id', 'object_type', 'profile_id']) ->cursorPaginate(6) ->withQueryString() ); diff --git a/app/Http/Controllers/AdminShadowFilterController.php b/app/Http/Controllers/AdminShadowFilterController.php new file mode 100644 index 000000000..461e1d0c2 --- /dev/null +++ b/app/Http/Controllers/AdminShadowFilterController.php @@ -0,0 +1,122 @@ +middleware(['auth','admin']); + } + + public function home(Request $request) + { + $filter = $request->input('filter'); + $searchQuery = $request->input('q'); + $filters = AdminShadowFilter::when($filter, function($q, $filter) { + if($filter == 'all') { + return $q; + } else if($filter == 'inactive') { + return $q->whereActive(false); + } else { + return $q; + } + }, function($q, $filter) { + return $q->whereActive(true); + }) + ->when($searchQuery, function($q, $searchQuery) { + $ids = Profile::where('username', 'like', '%' . $searchQuery . '%') + ->limit(100) + ->pluck('id') + ->toArray(); + return $q->where('item_type', 'App\Profile')->whereIn('item_id', $ids); + }) + ->latest() + ->paginate(10) + ->withQueryString(); + + return view('admin.asf.home', compact('filters')); + } + + public function create(Request $request) + { + return view('admin.asf.create'); + } + + public function edit(Request $request, $id) + { + $filter = AdminShadowFilter::findOrFail($id); + $profile = AccountService::get($filter->item_id); + return view('admin.asf.edit', compact('filter', 'profile')); + } + + public function store(Request $request) + { + $this->validate($request, [ + 'username' => 'required', + 'active' => 'sometimes', + 'note' => 'sometimes', + 'hide_from_public_feeds' => 'sometimes' + ]); + + $profile = Profile::whereUsername($request->input('username'))->first(); + + if(!$profile) { + return back()->withErrors(['Invalid account']); + } + + if($profile->user && $profile->user->is_admin) { + return back()->withErrors(['Cannot filter an admin account']); + } + + $active = $request->has('active') && $request->has('hide_from_public_feeds'); + + AdminShadowFilter::updateOrCreate([ + 'item_id' => $profile->id, + 'item_type' => get_class($profile) + ], [ + 'is_local' => $profile->domain === null, + 'note' => $request->input('note'), + 'hide_from_public_feeds' => $request->has('hide_from_public_feeds'), + 'admin_id' => $request->user()->profile_id, + 'active' => $active + ]); + + AdminShadowFilterService::refresh(); + + return redirect('/i/admin/asf/home'); + } + + public function storeEdit(Request $request, $id) + { + $this->validate($request, [ + 'active' => 'sometimes', + 'note' => 'sometimes', + 'hide_from_public_feeds' => 'sometimes' + ]); + + $filter = AdminShadowFilter::findOrFail($id); + + $profile = Profile::findOrFail($filter->item_id); + + if($profile->user && $profile->user->is_admin) { + return back()->withErrors(['Cannot filter an admin account']); + } + + $active = $request->has('active'); + $filter->active = $active; + $filter->hide_from_public_feeds = $request->has('hide_from_public_feeds'); + $filter->note = $request->input('note'); + $filter->save(); + + AdminShadowFilterService::refresh(); + + return redirect('/i/admin/asf/home'); + } +} diff --git a/app/Http/Controllers/Api/ApiV1Controller.php b/app/Http/Controllers/Api/ApiV1Controller.php index a0a240ba1..635cba24e 100644 --- a/app/Http/Controllers/Api/ApiV1Controller.php +++ b/app/Http/Controllers/Api/ApiV1Controller.php @@ -2193,6 +2193,7 @@ class ApiV1Controller extends Controller if($pid) { $status['favourited'] = (bool) LikeService::liked($pid, $s['id']); $status['reblogged'] = (bool) ReblogService::get($pid, $status['id']); + $status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']); } return $status; }) @@ -2203,6 +2204,7 @@ class ApiV1Controller extends Controller if(!empty($status['reblog'])) { $status['reblog']['favourited'] = (bool) LikeService::liked($pid, $status['reblog']['id']); $status['reblog']['reblogged'] = (bool) ReblogService::get($pid, $status['reblog']['id']); + $status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']); } return $status; @@ -2244,6 +2246,7 @@ class ApiV1Controller extends Controller if($pid) { $status['favourited'] = (bool) LikeService::liked($pid, $s['id']); $status['reblogged'] = (bool) ReblogService::get($pid, $status['id']); + $status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']); } return $status; }) @@ -2254,6 +2257,7 @@ class ApiV1Controller extends Controller if(!empty($status['reblog'])) { $status['reblog']['favourited'] = (bool) LikeService::liked($pid, $status['reblog']['id']); $status['reblog']['reblogged'] = (bool) ReblogService::get($pid, $status['reblog']['id']); + $status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']); } return $status; @@ -2378,6 +2382,7 @@ class ApiV1Controller extends Controller if($user) { $status['favourited'] = (bool) LikeService::liked($user->profile_id, $k); $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $status['id']); + $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $status['id']); } return $status; }) @@ -2502,7 +2507,7 @@ class ApiV1Controller extends Controller { abort_if(!$request->user(), 403); - $user = $request->user(); + $pid = $request->user()->profile_id; $res = $request->has(self::PF_API_ENTITY_KEY) ? StatusService::get($id, false) : StatusService::getMastodon($id, false); if(!$res || !isset($res['visibility'])) { @@ -2512,17 +2517,23 @@ class ApiV1Controller extends Controller $scope = $res['visibility']; if(!in_array($scope, ['public', 'unlisted'])) { if($scope === 'private') { - if(intval($res['account']['id']) !== intval($user->profile_id)) { - abort_unless(FollowerService::follows($user->profile_id, $res['account']['id']), 403); + if(intval($res['account']['id']) !== intval($pid)) { + abort_unless(FollowerService::follows($pid, $res['account']['id']), 403); } } else { abort(400, 'Invalid request'); } } - $res['favourited'] = LikeService::liked($user->profile_id, $res['id']); - $res['reblogged'] = ReblogService::get($user->profile_id, $res['id']); - $res['bookmarked'] = BookmarkService::get($user->profile_id, $res['id']); + if(!empty($res['reblog']) && isset($res['reblog']['id'])) { + $res['reblog']['favourited'] = (bool) LikeService::liked($pid, $res['reblog']['id']); + $res['reblog']['reblogged'] = (bool) ReblogService::get($pid, $res['reblog']['id']); + $res['reblog']['bookmarked'] = BookmarkService::get($pid, $res['reblog']['id']); + } + + $res['favourited'] = LikeService::liked($pid, $res['id']); + $res['reblogged'] = ReblogService::get($pid, $res['id']); + $res['bookmarked'] = BookmarkService::get($pid, $res['id']); return $this->json($res); } @@ -3615,8 +3626,8 @@ class ApiV1Controller extends Controller abort_if(!$request->user(), 403); $pid = $request->user()->profile_id; - $home = $request->input('home.last_read_id'); - $notifications = $request->input('notifications.last_read_id'); + $home = $request->input('home[last_read_id]'); + $notifications = $request->input('notifications[last_read_id]'); if($home) { return $this->json(MarkerService::set($pid, 'home', $home)); diff --git a/app/Http/Controllers/Api/ApiV2Controller.php b/app/Http/Controllers/Api/ApiV2Controller.php index 757e14dce..2ca5b96c5 100644 --- a/app/Http/Controllers/Api/ApiV2Controller.php +++ b/app/Http/Controllers/Api/ApiV2Controller.php @@ -34,6 +34,7 @@ use App\Transformer\Api\Mastodon\v1\{ use App\Transformer\Api\{ RelationshipTransformer, }; +use App\Util\Site\Nodeinfo; class ApiV2Controller extends Controller { @@ -77,12 +78,7 @@ class ApiV2Controller extends Controller 'description' => config_cache('app.short_description'), 'usage' => [ 'users' => [ - 'active_month' => (int) Cache::remember('api:nodeinfo:am', 172800, function() { - return User::select('last_active_at', 'created_at') - ->where('last_active_at', '>', now()->subMonths(1)) - ->orWhere('created_at', '>', now()->subMonths(1)) - ->count(); - }) + 'active_month' => (int) Nodeinfo::activeUsersMonthly() ] ], 'thumbnail' => [ diff --git a/app/Http/Controllers/ComposeController.php b/app/Http/Controllers/ComposeController.php index 54526ffe8..9be50f346 100644 --- a/app/Http/Controllers/ComposeController.php +++ b/app/Http/Controllers/ComposeController.php @@ -415,7 +415,7 @@ class ComposeController extends Controller $results = Profile::select('id','domain','username') ->whereNotIn('id', $blocked) ->where('username','like','%'.$q.'%') - ->groupBy('domain') + ->groupBy('id', 'domain') ->limit(15) ->get() ->map(function($profile) { diff --git a/app/Http/Controllers/RemoteAuthController.php b/app/Http/Controllers/RemoteAuthController.php index 72a2a08d5..e068f5d75 100644 --- a/app/Http/Controllers/RemoteAuthController.php +++ b/app/Http/Controllers/RemoteAuthController.php @@ -23,7 +23,13 @@ class RemoteAuthController extends Controller { public function start(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); if($request->user()) { return redirect('/'); } @@ -37,7 +43,13 @@ class RemoteAuthController extends Controller public function getAuthDomains(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); if(config('remote-auth.mastodon.domains.only_custom')) { $res = config('remote-auth.mastodon.domains.custom'); @@ -69,7 +81,14 @@ class RemoteAuthController extends Controller public function redirect(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); + $this->validate($request, ['domain' => 'required']); $domain = $request->input('domain'); @@ -158,6 +177,14 @@ class RemoteAuthController extends Controller public function preflight(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); + if(!$request->filled('d') || !$request->filled('dsh') || !$request->session()->exists('oauth_redirect_to')) { return redirect('/login'); } @@ -167,6 +194,14 @@ class RemoteAuthController extends Controller public function handleCallback(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); + $domain = $request->session()->get('oauth_domain'); if($request->filled('code')) { @@ -195,7 +230,13 @@ class RemoteAuthController extends Controller public function onboarding(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); if($request->user()) { return redirect('/'); } @@ -204,6 +245,13 @@ class RemoteAuthController extends Controller public function sessionCheck(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 403); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -248,6 +296,13 @@ class RemoteAuthController extends Controller public function sessionGetMastodonData(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 403); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -279,6 +334,13 @@ class RemoteAuthController extends Controller public function sessionValidateUsername(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 403); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -334,6 +396,13 @@ class RemoteAuthController extends Controller public function sessionValidateEmail(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 403); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -359,6 +428,13 @@ class RemoteAuthController extends Controller public function sessionGetMastodonFollowers(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); abort_unless($request->session()->exists('oauth_remasto_id'), 403); @@ -386,6 +462,13 @@ class RemoteAuthController extends Controller public function handleSubmit(Request $request) { + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); abort_unless($request->session()->exists('oauth_remasto_id'), 403); @@ -464,7 +547,13 @@ class RemoteAuthController extends Controller public function storeBio(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->user(), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -483,7 +572,13 @@ class RemoteAuthController extends Controller public function accountToId(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); @@ -525,7 +620,13 @@ class RemoteAuthController extends Controller public function storeAvatar(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->user(), 404); $this->validate($request, [ 'avatar_url' => 'required|active_url', @@ -547,7 +648,13 @@ class RemoteAuthController extends Controller public function finishUp(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_unless($request->user(), 404); $currentWebfinger = '@' . $request->user()->username . '@' . config('pixelfed.domain.app'); @@ -564,7 +671,13 @@ class RemoteAuthController extends Controller public function handleLogin(Request $request) { - abort_unless(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled'), 404); + abort_unless(( + config_cache('pixelfed.open_registration') && + config('remote-auth.mastodon.enabled') + ) || ( + config('remote-auth.mastodon.ignore_closed_state') && + config('remote-auth.mastodon.enabled') + ), 404); abort_if($request->user(), 404); abort_unless($request->session()->exists('oauth_domain'), 403); abort_unless($request->session()->exists('oauth_remote_session_token'), 403); diff --git a/app/Http/Controllers/Settings/PrivacySettings.php b/app/Http/Controllers/Settings/PrivacySettings.php index 3d1cd4515..9a5febe83 100644 --- a/app/Http/Controllers/Settings/PrivacySettings.php +++ b/app/Http/Controllers/Settings/PrivacySettings.php @@ -20,13 +20,13 @@ trait PrivacySettings public function privacy() { - $user = Auth::user(); - $settings = $user->settings; - $profile = $user->profile; - $is_private = $profile->is_private; - $settings['is_private'] = (bool) $is_private; + $user = Auth::user(); + $settings = $user->settings; + $profile = $user->profile; + $is_private = $profile->is_private; + $settings['is_private'] = (bool) $is_private; - return view('settings.privacy', compact('settings', 'profile')); + return view('settings.privacy', compact('settings', 'profile')); } public function privacyStore(Request $request) @@ -39,11 +39,13 @@ trait PrivacySettings 'public_dm', 'show_profile_follower_count', 'show_profile_following_count', + 'indexable', 'show_atom', ]; - $profile->is_suggestable = $request->input('is_suggestable') == 'on'; - $profile->save(); + $profile->indexable = $request->input('indexable') == 'on'; + $profile->is_suggestable = $request->input('is_suggestable') == 'on'; + $profile->save(); foreach ($fields as $field) { $form = $request->input($field); @@ -70,6 +72,8 @@ trait PrivacySettings } else { $settings->{$field} = false; } + } elseif ($field == 'indexable') { + } else { if ($form == 'on') { $settings->{$field} = true; diff --git a/app/Http/Controllers/Stories/StoryApiV1Controller.php b/app/Http/Controllers/Stories/StoryApiV1Controller.php index e32fffa26..db2b1f533 100644 --- a/app/Http/Controllers/Stories/StoryApiV1Controller.php +++ b/app/Http/Controllers/Stories/StoryApiV1Controller.php @@ -20,6 +20,7 @@ use App\Jobs\StoryPipeline\StoryViewDeliver; use App\Services\AccountService; use App\Services\MediaPathService; use App\Services\StoryService; +use App\Http\Resources\StoryView as StoryViewResource; class StoryApiV1Controller extends Controller { @@ -355,4 +356,26 @@ class StoryApiV1Controller extends Controller $path = $photo->storePubliclyAs($storagePath, Str::random(random_int(2, 12)) . '_' . Str::random(random_int(32, 35)) . '_' . Str::random(random_int(1, 14)) . '.' . $photo->extension()); return $path; } + + public function viewers(Request $request) + { + abort_if(!config_cache('instance.stories.enabled') || !$request->user(), 404); + + $this->validate($request, [ + 'sid' => 'required|string|min:1|max:50' + ]); + + $pid = $request->user()->profile_id; + $sid = $request->input('sid'); + + $story = Story::whereProfileId($pid) + ->whereActive(true) + ->findOrFail($sid); + + $viewers = StoryView::whereStoryId($story->id) + ->orderByDesc('id') + ->cursorPaginate(10); + + return StoryViewResource::collection($viewers); + } } diff --git a/app/Http/Resources/StoryView.php b/app/Http/Resources/StoryView.php new file mode 100644 index 000000000..891bf2eee --- /dev/null +++ b/app/Http/Resources/StoryView.php @@ -0,0 +1,20 @@ + + */ + public function toArray(Request $request) + { + return AccountService::get($this->profile_id, true); + } +} diff --git a/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php b/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php new file mode 100644 index 000000000..230797bf6 --- /dev/null +++ b/app/Jobs/AvatarPipeline/AvatarStorageCleanup.php @@ -0,0 +1,67 @@ +avatar->profile_id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-storage-cleanup:{$this->avatar->profile_id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct(Avatar $avatar) + { + $this->avatar = $avatar->withoutRelations(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + AvatarService::cleanup($this->avatar, true); + + return; + } +} diff --git a/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php b/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php new file mode 100644 index 000000000..f432e1e56 --- /dev/null +++ b/app/Jobs/AvatarPipeline/AvatarStorageLargePurge.php @@ -0,0 +1,80 @@ +avatar->profile_id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-storage-purge:{$this->avatar->profile_id}"))->shared()->dontRelease()]; + } + + /** + * Create a new job instance. + */ + public function __construct(Avatar $avatar) + { + $this->avatar = $avatar->withoutRelations(); + } + + /** + * Execute the job. + */ + public function handle(): void + { + $avatar = $this->avatar; + + $disk = AvatarService::disk(); + + $files = collect(AvatarService::storage($avatar)); + + $curFile = Str::of($avatar->cdn_url)->explode('/')->last(); + + $files = $files->filter(function($f) use($curFile) { + return !$curFile || !str_ends_with($f, $curFile); + })->each(function($name) use($disk) { + $disk->delete($name); + }); + + return; + } +} diff --git a/app/Jobs/AvatarPipeline/CreateAvatar.php b/app/Jobs/AvatarPipeline/CreateAvatar.php index fd5f94cc7..f773d1590 100644 --- a/app/Jobs/AvatarPipeline/CreateAvatar.php +++ b/app/Jobs/AvatarPipeline/CreateAvatar.php @@ -2,19 +2,25 @@ namespace App\Jobs\AvatarPipeline; -use App\Avatar; -use App\Profile; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\Middleware\WithoutOverlapping; +use App\Avatar; +use App\Profile; -class CreateAvatar implements ShouldQueue +class CreateAvatar implements ShouldQueue, ShouldBeUniqueUntilProcessing { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; - protected $profile; + public $profile; + public $tries = 3; + public $maxExceptions = 3; + public $timeout = 900; + public $failOnTimeout = true; /** * Delete the job if its models no longer exist. @@ -22,6 +28,31 @@ class CreateAvatar implements ShouldQueue * @var bool */ public $deleteWhenMissingModels = true; + + /** + * The number of seconds after which the job's unique lock will be released. + * + * @var int + */ + public $uniqueFor = 3600; + + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return 'avatar:create:' . $this->profile->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("avatar-create:{$this->profile->id}"))->shared()->dontRelease()]; + } /** * Create a new job instance. @@ -30,7 +61,7 @@ class CreateAvatar implements ShouldQueue */ public function __construct(Profile $profile) { - $this->profile = $profile; + $this->profile = $profile->withoutRelations(); } /** @@ -41,12 +72,18 @@ class CreateAvatar implements ShouldQueue public function handle() { $profile = $this->profile; + $isRemote = (bool) $profile->private_key == null; $path = 'public/avatars/default.jpg'; - $avatar = new Avatar(); - $avatar->profile_id = $profile->id; - $avatar->media_path = $path; - $avatar->change_count = 0; - $avatar->last_processed_at = \Carbon\Carbon::now(); - $avatar->save(); + Avatar::updateOrCreate( + [ + 'profile_id' => $profile->id, + ], + [ + 'media_path' => $path, + 'change_count' => 0, + 'is_remote' => $isRemote, + 'last_processed_at' => now() + ] + ); } } diff --git a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php index df972dd38..4e4a1b2ec 100644 --- a/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php +++ b/app/Jobs/AvatarPipeline/RemoteAvatarFetch.php @@ -108,7 +108,7 @@ class RemoteAvatarFetch implements ShouldQueue $avatar->remote_url = $icon['url']; $avatar->save(); - MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false); + MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true); return 1; } diff --git a/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php b/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php index 259058385..c8c6820e4 100644 --- a/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php +++ b/app/Jobs/AvatarPipeline/RemoteAvatarFetchFromUrl.php @@ -89,7 +89,6 @@ class RemoteAvatarFetchFromUrl implements ShouldQueue $avatar->save(); } - MediaStorageService::avatar($avatar, boolval(config_cache('pixelfed.cloud_storage')) == false, true); return 1; diff --git a/app/Jobs/FollowPipeline/FollowServiceWarmCache.php b/app/Jobs/FollowPipeline/FollowServiceWarmCache.php index cabea9958..990236f69 100644 --- a/app/Jobs/FollowPipeline/FollowServiceWarmCache.php +++ b/app/Jobs/FollowPipeline/FollowServiceWarmCache.php @@ -8,10 +8,13 @@ use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\Middleware\WithoutOverlapping; use App\Services\AccountService; use App\Services\FollowerService; use Cache; use DB; +use Storage; +use App\Follower; use App\Profile; class FollowServiceWarmCache implements ShouldQueue @@ -23,6 +26,16 @@ class FollowServiceWarmCache implements ShouldQueue public $timeout = 5000; public $failOnTimeout = false; + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping($this->profileId))->dontRelease()]; + } + /** * Create a new job instance. * @@ -42,6 +55,10 @@ class FollowServiceWarmCache implements ShouldQueue { $id = $this->profileId; + if(Cache::has(FollowerService::FOLLOWERS_SYNC_KEY . $id) && Cache::has(FollowerService::FOLLOWING_SYNC_KEY . $id)) { + return; + } + $account = AccountService::get($id, true); if(!$account) { @@ -50,25 +67,43 @@ class FollowServiceWarmCache implements ShouldQueue return; } - DB::table('followers') - ->select('id', 'following_id', 'profile_id') - ->whereFollowingId($id) - ->orderBy('id') - ->chunk(200, function($followers) use($id) { - foreach($followers as $follow) { - FollowerService::add($follow->profile_id, $id); - } - }); + $hasFollowerPostProcessing = false; + $hasFollowingPostProcessing = false; - DB::table('followers') - ->select('id', 'following_id', 'profile_id') - ->whereProfileId($id) - ->orderBy('id') - ->chunk(200, function($followers) use($id) { - foreach($followers as $follow) { - FollowerService::add($id, $follow->following_id); - } - }); + if(Follower::whereProfileId($id)->orWhere('following_id', $id)->count()) { + $following = []; + $followers = []; + foreach(Follower::lazy() as $follow) { + if($follow->following_id != $id && $follow->profile_id != $id) { + continue; + } + if($follow->profile_id == $id) { + $following[] = $follow->following_id; + } else { + $followers[] = $follow->profile_id; + } + } + + if(count($followers) > 100) { + // store follower ids and process in another job + Storage::put('follow-warm-cache/' . $id . '/followers.json', json_encode($followers)); + $hasFollowerPostProcessing = true; + } else { + foreach($followers as $follower) { + FollowerService::add($follower, $id); + } + } + + if(count($following) > 100) { + // store following ids and process in another job + Storage::put('follow-warm-cache/' . $id . '/following.json', json_encode($following)); + $hasFollowingPostProcessing = true; + } else { + foreach($following as $following) { + FollowerService::add($id, $following); + } + } + } Cache::put(FollowerService::FOLLOWERS_SYNC_KEY . $id, 1, 604800); Cache::put(FollowerService::FOLLOWING_SYNC_KEY . $id, 1, 604800); @@ -82,6 +117,14 @@ class FollowServiceWarmCache implements ShouldQueue AccountService::del($id); + if($hasFollowingPostProcessing) { + FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'following')->onQueue('follow'); + } + + if($hasFollowerPostProcessing) { + FollowServiceWarmCacheLargeIngestPipeline::dispatch($id, 'followers')->onQueue('follow'); + } + return; } } diff --git a/app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php b/app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php new file mode 100644 index 000000000..3299bf7a4 --- /dev/null +++ b/app/Jobs/FollowPipeline/FollowServiceWarmCacheLargeIngestPipeline.php @@ -0,0 +1,88 @@ +profileId = $profileId; + $this->followType = $followType; + } + + /** + * Execute the job. + * + * @return void + */ + public function handle() + { + $pid = $this->profileId; + $type = $this->followType; + + if($type === 'followers') { + $key = 'follow-warm-cache/' . $pid . '/followers.json'; + if(!Storage::exists($key)) { + return; + } + $file = Storage::get($key); + $json = json_decode($file, true); + + foreach($json as $id) { + FollowerService::add($id, $pid, false); + usleep(random_int(500, 3000)); + } + sleep(5); + Storage::delete($key); + } + + if($type === 'following') { + $key = 'follow-warm-cache/' . $pid . '/following.json'; + if(!Storage::exists($key)) { + return; + } + $file = Storage::get($key); + $json = json_decode($file, true); + + foreach($json as $id) { + FollowerService::add($pid, $id, false); + usleep(random_int(500, 3000)); + } + sleep(5); + Storage::delete($key); + } + + sleep(random_int(2, 5)); + $files = Storage::files('follow-warm-cache/' . $pid); + if(empty($files)) { + Storage::deleteDirectory('follow-warm-cache/' . $pid); + } + } +} diff --git a/app/Jobs/ProfilePipeline/DecrementPostCount.php b/app/Jobs/ProfilePipeline/DecrementPostCount.php index d6781d7a5..b463f1dda 100644 --- a/app/Jobs/ProfilePipeline/DecrementPostCount.php +++ b/app/Jobs/ProfilePipeline/DecrementPostCount.php @@ -43,15 +43,9 @@ class DecrementPostCount implements ShouldQueue return 1; } - if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) { - $profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count(); - $profile->save(); - AccountService::del($id); - } else { - $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0; - $profile->save(); - AccountService::del($id); - } + $profile->status_count = $profile->status_count ? $profile->status_count - 1 : 0; + $profile->save(); + AccountService::del($id); return 1; } diff --git a/app/Jobs/ProfilePipeline/IncrementPostCount.php b/app/Jobs/ProfilePipeline/IncrementPostCount.php index 9c7585e25..fe8d90648 100644 --- a/app/Jobs/ProfilePipeline/IncrementPostCount.php +++ b/app/Jobs/ProfilePipeline/IncrementPostCount.php @@ -43,17 +43,10 @@ class IncrementPostCount implements ShouldQueue return 1; } - if($profile->updated_at && $profile->updated_at->lt(now()->subDays(30))) { - $profile->status_count = Status::whereProfileId($id)->whereNull(['in_reply_to_id', 'reblog_of_id'])->count(); - $profile->last_status_at = now(); - $profile->save(); - AccountService::del($id); - } else { - $profile->status_count = $profile->status_count + 1; - $profile->last_status_at = now(); - $profile->save(); - AccountService::del($id); - } + $profile->status_count = $profile->status_count + 1; + $profile->last_status_at = now(); + $profile->save(); + AccountService::del($id); return 1; } diff --git a/app/Jobs/StatusPipeline/RemoteStatusDelete.php b/app/Jobs/StatusPipeline/RemoteStatusDelete.php index 19c17b54c..aabb81755 100644 --- a/app/Jobs/StatusPipeline/RemoteStatusDelete.php +++ b/app/Jobs/StatusPipeline/RemoteStatusDelete.php @@ -21,9 +21,11 @@ use App\{ }; use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; +use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Queue\Middleware\WithoutOverlapping; use League\Fractal; use Illuminate\Support\Str; use League\Fractal\Serializer\ArraySerializer; @@ -37,8 +39,9 @@ use App\Services\AccountService; use App\Services\CollectionService; use App\Services\StatusService; use App\Jobs\MediaPipeline\MediaDeletePipeline; +use App\Jobs\ProfilePipeline\DecrementPostCount; -class RemoteStatusDelete implements ShouldQueue +class RemoteStatusDelete implements ShouldQueue, ShouldBeUniqueUntilProcessing { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; @@ -51,9 +54,35 @@ class RemoteStatusDelete implements ShouldQueue */ public $deleteWhenMissingModels = true; - public $timeout = 90; - public $tries = 2; - public $maxExceptions = 1; + public $tries = 3; + public $maxExceptions = 3; + public $timeout = 180; + public $failOnTimeout = true; + + /** + * The number of seconds after which the job's unique lock will be released. + * + * @var int + */ + public $uniqueFor = 3600; + + /** + * Get the unique ID for the job. + */ + public function uniqueId(): string + { + return 'status:remote:delete:' . $this->status->id; + } + + /** + * Get the middleware the job should pass through. + * + * @return array + */ + public function middleware(): array + { + return [(new WithoutOverlapping("status-remote-delete-{$this->status->id}"))->shared()->dontRelease()]; + } /** * Create a new job instance. @@ -62,7 +91,7 @@ class RemoteStatusDelete implements ShouldQueue */ public function __construct(Status $status) { - $this->status = $status; + $this->status = $status->withoutRelations(); } /** @@ -77,14 +106,10 @@ class RemoteStatusDelete implements ShouldQueue if($status->deleted_at) { return; } - $profile = $this->status->profile; StatusService::del($status->id, true); - if($profile->status_count && $profile->status_count > 0) { - $profile->status_count = $profile->status_count - 1; - $profile->save(); - } + DecrementPostCount::dispatch($status->profile_id)->onQueue('inbox'); return $this->unlinkRemoveMedia($status); } diff --git a/app/Jobs/StatusPipeline/StatusEntityLexer.php b/app/Jobs/StatusPipeline/StatusEntityLexer.php index d205f1e21..2bbc92102 100644 --- a/app/Jobs/StatusPipeline/StatusEntityLexer.php +++ b/app/Jobs/StatusPipeline/StatusEntityLexer.php @@ -20,6 +20,7 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; use App\Services\UserFilterService; +use App\Services\AdminShadowFilterService; class StatusEntityLexer implements ShouldQueue { @@ -176,7 +177,9 @@ class StatusEntityLexer implements ShouldQueue $status->reblog_of_id === null && ($hideNsfw ? $status->is_nsfw == false : true) ) { - PublicTimelineService::add($status->id); + if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) { + PublicTimelineService::add($status->id); + } } if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') { diff --git a/app/Jobs/StatusPipeline/StatusTagsPipeline.php b/app/Jobs/StatusPipeline/StatusTagsPipeline.php index a72e6d50e..893fa6a83 100644 --- a/app/Jobs/StatusPipeline/StatusTagsPipeline.php +++ b/app/Jobs/StatusPipeline/StatusTagsPipeline.php @@ -45,6 +45,11 @@ class StatusTagsPipeline implements ShouldQueue { $res = $this->activity; $status = $this->status; + + if(isset($res['tag']['type'], $res['tag']['name'])) { + $res['tag'] = [$res['tag']]; + } + $tags = collect($res['tag']); // Emoji @@ -73,19 +78,18 @@ class StatusTagsPipeline implements ShouldQueue if(config('database.default') === 'pgsql') { $hashtag = Hashtag::where('name', 'ilike', $name) - ->orWhere('slug', 'ilike', str_slug($name)) + ->orWhere('slug', 'ilike', str_slug($name, '-', false)) ->first(); - if(!$hashtag) { - $hashtag = new Hashtag; - $hashtag->name = $name; - $hashtag->slug = str_slug($name); - $hashtag->save(); - } + if(!$hashtag) { + $hashtag = Hashtag::updateOrCreate([ + 'slug' => str_slug($name, '-', false), + 'name' => $name + ]); + } } else { - $hashtag = Hashtag::firstOrCreate([ - 'slug' => str_slug($name) - ], [ + $hashtag = Hashtag::updateOrCreate([ + 'slug' => str_slug($name, '-', false), 'name' => $name ]); } diff --git a/app/Models/AdminShadowFilter.php b/app/Models/AdminShadowFilter.php new file mode 100644 index 000000000..f98086f7f --- /dev/null +++ b/app/Models/AdminShadowFilter.php @@ -0,0 +1,27 @@ + 'datetime' + ]; + + public function account() + { + if($this->item_type === 'App\Profile') { + return AccountService::get($this->item_id, true); + } + + return; + } +} diff --git a/app/Services/AdminShadowFilterService.php b/app/Services/AdminShadowFilterService.php new file mode 100644 index 000000000..a5933508a --- /dev/null +++ b/app/Services/AdminShadowFilterService.php @@ -0,0 +1,51 @@ +whereActive(1) + ->where('hide_from_public_feeds', true) + ->pluck('item_id') + ->toArray(); + } + + public static function getHideFromPublicFeedsList($refresh = false) + { + $key = self::CACHE_KEY . 'list:hide_from_public_feeds'; + if($refresh) { + Cache::forget($key); + } + return Cache::remember($key, 86400, function() { + return AdminShadowFilter::whereItemType('App\Profile') + ->whereActive(1) + ->where('hide_from_public_feeds', true) + ->pluck('item_id') + ->toArray(); + }); + } + + public static function canAddToPublicFeedByProfileId($profileId) + { + return !in_array($profileId, self::getHideFromPublicFeedsList()); + } + + public static function refresh() + { + $keys = [ + self::CACHE_KEY . 'list:hide_from_public_feeds' + ]; + + foreach($keys as $key) { + Cache::forget($key); + } + } +} diff --git a/app/Services/AvatarService.php b/app/Services/AvatarService.php index 1c5e9e0c1..af578fdef 100644 --- a/app/Services/AvatarService.php +++ b/app/Services/AvatarService.php @@ -3,21 +3,125 @@ namespace App\Services; use Cache; +use Storage; +use Illuminate\Support\Str; +use App\Avatar; use App\Profile; +use App\Jobs\AvatarPipeline\AvatarStorageLargePurge; +use League\Flysystem\UnableToCheckDirectoryExistence; +use League\Flysystem\UnableToRetrieveMetadata; class AvatarService { - public static function get($profile_id) - { - $exists = Cache::get('avatar:' . $profile_id); - if($exists) { - return $exists; - } + public static function get($profile_id) + { + $exists = Cache::get('avatar:' . $profile_id); + if($exists) { + return $exists; + } - $profile = Profile::find($profile_id); - if(!$profile) { - return config('app.url') . '/storage/avatars/default.jpg'; - } - return $profile->avatarUrl(); - } + $profile = Profile::find($profile_id); + if(!$profile) { + return config('app.url') . '/storage/avatars/default.jpg'; + } + return $profile->avatarUrl(); + } + + public static function disk() + { + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + return false; + } + + $driver = $storage['cloud'] == false ? 'local' : config('filesystems.cloud'); + $disk = Storage::disk($driver); + + return $disk; + } + + public static function storage(Avatar $avatar) + { + $disk = self::disk(); + + if(!$disk) { + return; + } + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/'; + + return $disk->allFiles($base . $avatar->profile_id); + } + + public static function cleanup($avatar, $confirm = false) + { + if(!$avatar || !$confirm) { + return; + } + + if($avatar->cdn_url == null) { + return; + } + + $storage = [ + 'cloud' => boolval(config_cache('pixelfed.cloud_storage')), + 'local' => boolval(config_cache('federation.avatars.store_local')) + ]; + + if(!$storage['cloud'] && !$storage['local']) { + return; + } + + $disk = self::disk(); + + if(!$disk) { + return; + } + + $base = ($storage['cloud'] == false ? 'public/cache/' : 'cache/') . 'avatars/'; + + try { + $exists = $disk->directoryExists($base . $avatar->profile_id); + } catch ( + UnableToRetrieveMetadata | + UnableToCheckDirectoryExistence | + Exception $e + ) { + return; + } + + if(!$exists) { + return; + } + + $files = collect($disk->allFiles($base . $avatar->profile_id)); + + if(!$files || !$files->count() || $files->count() === 1) { + return; + } + + if($files->count() > 5) { + AvatarStorageLargePurge::dispatch($avatar)->onQueue('mmo'); + return; + } + + $curFile = Str::of($avatar->cdn_url)->explode('/')->last(); + + $files = $files->filter(function($f) use($curFile) { + return !$curFile || !str_ends_with($f, $curFile); + })->each(function($name) use($disk) { + $disk->delete($name); + }); + + return; + } } diff --git a/app/Services/FollowerService.php b/app/Services/FollowerService.php index 9398fa53f..1c00a6f49 100644 --- a/app/Services/FollowerService.php +++ b/app/Services/FollowerService.php @@ -20,10 +20,14 @@ class FollowerService const FOLLOWING_KEY = 'pf:services:follow:following:id:'; const FOLLOWERS_KEY = 'pf:services:follow:followers:id:'; - public static function add($actor, $target) + public static function add($actor, $target, $refresh = true) { $ts = (int) microtime(true); - RelationshipService::refresh($actor, $target); + if($refresh) { + RelationshipService::refresh($actor, $target); + } else { + RelationshipService::forget($actor, $target); + } Redis::zadd(self::FOLLOWING_KEY . $actor, $ts, $target); Redis::zadd(self::FOLLOWERS_KEY . $target, $ts, $actor); Cache::forget('profile:following:' . $actor); diff --git a/app/Services/InstanceService.php b/app/Services/InstanceService.php index 1cead8d48..2ad991063 100644 --- a/app/Services/InstanceService.php +++ b/app/Services/InstanceService.php @@ -120,6 +120,9 @@ class InstanceService $pixels[] = $row; } + // Free the allocated GdImage object from memory: + imagedestroy($image); + $components_x = 4; $components_y = 4; $blurhash = Blurhash::encode($pixels, $components_x, $components_y); diff --git a/app/Services/LandingService.php b/app/Services/LandingService.php index 2a5acda07..ba16af5b6 100644 --- a/app/Services/LandingService.php +++ b/app/Services/LandingService.php @@ -9,17 +9,13 @@ use Illuminate\Support\Facades\Redis; use App\Status; use App\User; use App\Services\AccountService; +use App\Util\Site\Nodeinfo; class LandingService { public static function get($json = true) { - $activeMonth = Cache::remember('api:nodeinfo:am', 172800, function() { - return User::select('last_active_at') - ->where('last_active_at', '>', now()->subMonths(1)) - ->orWhere('created_at', '>', now()->subMonths(1)) - ->count(); - }); + $activeMonth = Nodeinfo::activeUsersMonthly(); $totalUsers = Cache::remember('api:nodeinfo:users', 43200, function() { return User::count(); diff --git a/app/Services/MediaStorageService.php b/app/Services/MediaStorageService.php index b52662d1f..128001de2 100644 --- a/app/Services/MediaStorageService.php +++ b/app/Services/MediaStorageService.php @@ -17,6 +17,7 @@ use App\Http\Controllers\AvatarController; use GuzzleHttp\Exception\RequestException; use App\Jobs\MediaPipeline\MediaDeletePipeline; use Illuminate\Support\Arr; +use App\Jobs\AvatarPipeline\AvatarStorageCleanup; class MediaStorageService { @@ -29,9 +30,9 @@ class MediaStorageService { return; } - public static function avatar($avatar, $local = false) + public static function avatar($avatar, $local = false, $skipRecentCheck = false) { - return (new self())->fetchAvatar($avatar, $local); + return (new self())->fetchAvatar($avatar, $local, $skipRecentCheck); } public static function head($url) @@ -86,12 +87,11 @@ class MediaStorageService { $thumbname = array_pop($pt); $storagePath = implode('/', $p); - $disk = Storage::disk(config('filesystems.cloud')); - $file = $disk->putFileAs($storagePath, new File($path), $name, 'public'); - $url = $disk->url($file); - $thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public'); - $thumbUrl = $disk->url($thumbFile); - $media->thumbnail_url = $thumbUrl; + $url = ResilientMediaStorageService::store($storagePath, $path, $name); + if($thumb) { + $thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname); + $media->thumbnail_url = $thumbUrl; + } $media->cdn_url = $url; $media->optimized_url = $url; $media->replicated_at = now(); @@ -183,6 +183,7 @@ class MediaStorageService { protected function fetchAvatar($avatar, $local = false, $skipRecentCheck = false) { + $queue = random_int(1, 15) > 5 ? 'mmo' : 'low'; $url = $avatar->remote_url; $driver = $local ? 'local' : config('filesystems.cloud'); @@ -206,7 +207,7 @@ class MediaStorageService { $max_size = (int) config('pixelfed.max_avatar_size') * 1000; if(!$skipRecentCheck) { - if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subDay())) { + if($avatar->last_fetched_at && $avatar->last_fetched_at->gt(now()->subMonths(3))) { return; } } @@ -262,6 +263,7 @@ class MediaStorageService { Cache::forget('avatar:' . $avatar->profile_id); AccountService::del($avatar->profile_id); + AvatarStorageCleanup::dispatch($avatar)->onQueue($queue)->delay(now()->addMinutes(random_int(3, 15))); unlink($tmpName); } diff --git a/app/Services/NotificationService.php b/app/Services/NotificationService.php index 139b13a69..d088c2015 100644 --- a/app/Services/NotificationService.php +++ b/app/Services/NotificationService.php @@ -16,6 +16,8 @@ use League\Fractal\Pagination\IlluminatePaginatorAdapter; class NotificationService { const CACHE_KEY = 'pf:services:notifications:ids:'; + const EPOCH_CACHE_KEY = 'pf:services:notifications:epoch-id:by-months:'; + const ITEM_CACHE_TTL = 86400; const MASTODON_TYPES = [ 'follow', 'follow_request', @@ -44,11 +46,22 @@ class NotificationService { return $res; } + public static function getEpochId($months = 6) + { + return Cache::remember(self::EPOCH_CACHE_KEY . $months, 1209600, function() use($months) { + if(Notification::count() === 0) { + return 0; + } + return Notification::where('created_at', '>', now()->subMonths($months))->first()->id; + }); + } + public static function coldGet($id, $start = 0, $stop = 400) { $stop = $stop > 400 ? 400 : $stop; - $ids = Notification::whereProfileId($id) - ->latest() + $ids = Notification::where('id', '>', self::getEpochId()) + ->where('profile_id', $id) + ->orderByDesc('id') ->skip($start) ->take($stop) ->pluck('id'); @@ -227,7 +240,7 @@ class NotificationService { public static function getNotification($id) { - $notification = Cache::remember('service:notification:'.$id, 86400, function() use($id) { + $notification = Cache::remember('service:notification:'.$id, self::ITEM_CACHE_TTL, function() use($id) { $n = Notification::with('item')->find($id); if(!$n) { @@ -259,19 +272,20 @@ class NotificationService { public static function setNotification(Notification $notification) { - return Cache::remember('service:notification:'.$notification->id, now()->addDays(3), function() use($notification) { + return Cache::remember('service:notification:'.$notification->id, self::ITEM_CACHE_TTL, function() use($notification) { $fractal = new Fractal\Manager(); $fractal->setSerializer(new ArraySerializer()); $resource = new Fractal\Resource\Item($notification, new NotificationTransformer()); return $fractal->createData($resource)->toArray(); }); - } + } public static function warmCache($id, $stop = 400, $force = false) { if(self::count($id) == 0 || $force == true) { - $ids = Notification::whereProfileId($id) - ->latest() + $ids = Notification::where('profile_id', $id) + ->where('id', '>', self::getEpochId()) + ->orderByDesc('id') ->limit($stop) ->pluck('id'); foreach($ids as $key) { diff --git a/app/Services/PublicTimelineService.php b/app/Services/PublicTimelineService.php index f2658e4b1..7cd6816b3 100644 --- a/app/Services/PublicTimelineService.php +++ b/app/Services/PublicTimelineService.php @@ -95,7 +95,7 @@ class PublicTimelineService { if(self::count() == 0 || $force == true) { $hideNsfw = config('instance.hide_nsfw_on_public_feeds'); Redis::del(self::CACHE_KEY); - $minId = SnowflakeService::byDate(now()->subDays(14)); + $minId = SnowflakeService::byDate(now()->subDays(90)); $ids = Status::where('id', '>', $minId) ->whereNull(['uri', 'in_reply_to_id', 'reblog_of_id']) ->when($hideNsfw, function($q, $hideNsfw) { @@ -105,9 +105,11 @@ class PublicTimelineService { ->whereScope('public') ->orderByDesc('id') ->limit($limit) - ->pluck('id'); - foreach($ids as $id) { - self::add($id); + ->pluck('id', 'profile_id'); + foreach($ids as $k => $id) { + if(AdminShadowFilterService::canAddToPublicFeedByProfileId($k)) { + self::add($id); + } } return 1; } diff --git a/app/Services/RelationshipService.php b/app/Services/RelationshipService.php index 3c6d2818f..476c9c9ae 100644 --- a/app/Services/RelationshipService.php +++ b/app/Services/RelationshipService.php @@ -66,6 +66,14 @@ class RelationshipService return self::get($aid, $tid); } + public static function forget($aid, $tid) + { + Cache::forget('pf:services:follower:audience:' . $aid); + Cache::forget('pf:services:follower:audience:' . $tid); + self::delete($tid, $aid); + self::delete($aid, $tid); + } + public static function defaultRelation($tid) { return [ diff --git a/app/Services/ResilientMediaStorageService.php b/app/Services/ResilientMediaStorageService.php new file mode 100644 index 000000000..ac1b089af --- /dev/null +++ b/app/Services/ResilientMediaStorageService.php @@ -0,0 +1,66 @@ +putFileAs($storagePath, new File($path), $name, 'public'); + return $disk->url($file); + }, random_int(100, 500)); + } + + public static function handleResilientStore($storagePath, $path, $name) + { + $attempts = 0; + return retry(4, function() use($storagePath, $path, $name, $attempts) { + self::$attempts++; + usleep(100000); + $baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud'); + try { + $disk = Storage::disk($baseDisk); + $file = $disk->putFileAs($storagePath, new File($path), $name, 'public'); + } catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {} + return $disk->url($file); + }, function (int $attempt, Exception $exception) { + return $attempt * 200; + }); + } + + public static function getAltDriver() + { + $drivers = []; + if(config('filesystems.disks.alt-primary.enabled')) { + $drivers[] = 'alt-primary'; + } + if(config('filesystems.disks.alt-secondary.enabled')) { + $drivers[] = 'alt-secondary'; + } + if(empty($drivers)) { + return false; + } + $key = array_rand($drivers, 1); + return $drivers[$key]; + } +} diff --git a/app/Services/StatusService.php b/app/Services/StatusService.php index bdf2f31db..4051bede4 100644 --- a/app/Services/StatusService.php +++ b/app/Services/StatusService.php @@ -22,9 +22,9 @@ class StatusService return self::CACHE_KEY . $p . $id; } - public static function get($id, $publicOnly = true) + public static function get($id, $publicOnly = true, $mastodonMode = false) { - return Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) { + $res = Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) { if($publicOnly) { $status = Status::whereScope('public')->find($id); } else { @@ -36,13 +36,23 @@ class StatusService $fractal = new Fractal\Manager(); $fractal->setSerializer(new ArraySerializer()); $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer()); - return $fractal->createData($resource)->toArray(); + $res = $fractal->createData($resource)->toArray(); + $res['_pid'] = isset($res['account']) && isset($res['account']['id']) ? $res['account']['id'] : null; + if(isset($res['_pid'])) { + unset($res['account']); + } + return $res; }); + if($res && isset($res['_pid'])) { + $res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true); + unset($res['_pid']); + } + return $res; } public static function getMastodon($id, $publicOnly = true) { - $status = self::get($id, $publicOnly); + $status = self::get($id, $publicOnly, true); if(!$status) { return null; } @@ -151,8 +161,6 @@ class StatusService } Cache::forget('status:transformer:media:attachments:' . $id); MediaService::del($id); - Cache::forget('status:thumb:nsfw0' . $id); - Cache::forget('status:thumb:nsfw1' . $id); Cache::forget('pf:services:sh:id:' . $id); PublicTimelineService::rem($id); NetworkTimelineService::rem($id); diff --git a/app/Status.php b/app/Status.php index 77262597e..d665464ae 100644 --- a/app/Status.php +++ b/app/Status.php @@ -9,7 +9,9 @@ use App\Http\Controllers\StatusController; use Illuminate\Database\Eloquent\SoftDeletes; use App\Models\Poll; use App\Services\AccountService; +use App\Services\StatusService; use App\Models\StatusEdit; +use Illuminate\Support\Str; class Status extends Model { @@ -95,16 +97,30 @@ class Status extends Model public function thumb($showNsfw = false) { - $key = $showNsfw ? 'status:thumb:nsfw1'.$this->id : 'status:thumb:nsfw0'.$this->id; - return Cache::remember($key, now()->addMinutes(15), function() use ($showNsfw) { - $type = $this->type ?? $this->setType(); - $is_nsfw = !$showNsfw ? $this->is_nsfw : false; - if ($this->media->count() == 0 || $is_nsfw || !in_array($type,['photo', 'photo:album', 'video'])) { - return url(Storage::url('public/no-preview.png')); - } + $entity = StatusService::get($this->id, false); - return url(Storage::url($this->firstMedia()->thumbnail_path)); - }); + if(!$entity || !isset($entity['media_attachments']) || empty($entity['media_attachments'])) { + return url(Storage::url('public/no-preview.png')); + } + + if((!isset($entity['sensitive']) || $entity['sensitive']) && !$showNsfw) { + return url(Storage::url('public/no-preview.png')); + } + + if(!isset($entity['visibility']) || !in_array($entity['visibility'], ['public', 'unlisted'])) { + return url(Storage::url('public/no-preview.png')); + } + + return collect($entity['media_attachments']) + ->filter(fn($media) => $media['type'] == 'image' && in_array($media['mime'], ['image/jpeg', 'image/png'])) + ->map(function($media) { + if(!Str::endsWith($media['preview_url'], ['no-preview.png', 'no-preview.jpg'])) { + return $media['preview_url']; + } + + return $media['url']; + }) + ->first() ?? url(Storage::url('public/no-preview.png')); } public function url($forceLocal = false) diff --git a/app/Transformer/ActivityPub/ProfileTransformer.php b/app/Transformer/ActivityPub/ProfileTransformer.php index 1df7b6100..cdd4eb82d 100644 --- a/app/Transformer/ActivityPub/ProfileTransformer.php +++ b/app/Transformer/ActivityPub/ProfileTransformer.php @@ -15,6 +15,7 @@ class ProfileTransformer extends Fractal\TransformerAbstract 'https://w3id.org/security/v1', 'https://www.w3.org/ns/activitystreams', [ + 'toot' => 'http://joinmastodon.org/ns#', 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers', 'alsoKnownAs' => [ '@id' => 'as:alsoKnownAs', @@ -23,7 +24,8 @@ class ProfileTransformer extends Fractal\TransformerAbstract 'movedTo' => [ '@id' => 'as:movedTo', '@type' => '@id' - ] + ], + 'indexable' => 'toot:indexable', ], ], 'id' => $profile->permalink(), @@ -37,6 +39,7 @@ class ProfileTransformer extends Fractal\TransformerAbstract 'summary' => $profile->bio, 'url' => $profile->url(), 'manuallyApprovesFollowers' => (bool) $profile->is_private, + 'indexable' => (bool) $profile->indexable, 'publicKey' => [ 'id' => $profile->permalink().'#main-key', 'owner' => $profile->permalink(), diff --git a/app/Transformer/ActivityPub/Verb/CreateNote.php b/app/Transformer/ActivityPub/Verb/CreateNote.php index a9d40d9ed..55fdfa8f4 100644 --- a/app/Transformer/ActivityPub/Verb/CreateNote.php +++ b/app/Transformer/ActivityPub/Verb/CreateNote.php @@ -81,7 +81,8 @@ class CreateNote extends Fractal\TransformerAbstract '@type' => '@id' ], 'toot' => 'http://joinmastodon.org/ns#', - 'Emoji' => 'toot:Emoji' + 'Emoji' => 'toot:Emoji', + 'blurhash' => 'toot:blurhash', ] ], 'id' => $status->permalink(), @@ -103,12 +104,22 @@ class CreateNote extends Fractal\TransformerAbstract 'cc' => $status->scopeToAudience('cc'), 'sensitive' => (bool) $status->is_nsfw, 'attachment' => $status->media()->orderBy('order')->get()->map(function ($media) { - return [ + $res = [ 'type' => $media->activityVerb(), 'mediaType' => $media->mime, 'url' => $media->url(), 'name' => $media->caption, ]; + if($media->blurhash) { + $res['blurhash'] = $media->blurhash; + } + if($media->width) { + $res['width'] = $media->width; + } + if($media->height) { + $res['height'] = $media->height; + } + return $res; })->toArray(), 'tag' => $tags, 'commentsEnabled' => (bool) !$status->comments_disabled, diff --git a/app/Transformer/ActivityPub/Verb/Note.php b/app/Transformer/ActivityPub/Verb/Note.php index 777bd22b0..1350641d4 100644 --- a/app/Transformer/ActivityPub/Verb/Note.php +++ b/app/Transformer/ActivityPub/Verb/Note.php @@ -82,7 +82,8 @@ class Note extends Fractal\TransformerAbstract '@type' => '@id' ], 'toot' => 'http://joinmastodon.org/ns#', - 'Emoji' => 'toot:Emoji' + 'Emoji' => 'toot:Emoji', + 'blurhash' => 'toot:blurhash', ] ], 'id' => $status->url(), @@ -97,12 +98,22 @@ class Note extends Fractal\TransformerAbstract 'cc' => $status->scopeToAudience('cc'), 'sensitive' => (bool) $status->is_nsfw, 'attachment' => $status->media()->orderBy('order')->get()->map(function ($media) { - return [ + $res = [ 'type' => $media->activityVerb(), 'mediaType' => $media->mime, 'url' => $media->url(), 'name' => $media->caption, ]; + if($media->blurhash) { + $res['blurhash'] = $media->blurhash; + } + if($media->width) { + $res['width'] = $media->width; + } + if($media->height) { + $res['height'] = $media->height; + } + return $res; })->toArray(), 'tag' => $tags, 'commentsEnabled' => (bool) !$status->comments_disabled, diff --git a/app/Transformer/Api/StatusStatelessTransformer.php b/app/Transformer/Api/StatusStatelessTransformer.php index c21720509..3c2c02d60 100644 --- a/app/Transformer/Api/StatusStatelessTransformer.php +++ b/app/Transformer/Api/StatusStatelessTransformer.php @@ -16,6 +16,7 @@ use App\Services\StatusLabelService; use App\Services\StatusMentionService; use App\Services\PollService; use App\Models\CustomEmoji; +use App\Util\Lexer\Autolink; class StatusStatelessTransformer extends Fractal\TransformerAbstract { @@ -23,6 +24,9 @@ class StatusStatelessTransformer extends Fractal\TransformerAbstract { $taggedPeople = MediaTagService::get($status->id); $poll = $status->type === 'poll' ? PollService::get($status->id) : null; + $rendered = config('exp.autolink') ? + ( $status->caption ? Autolink::create()->autolink($status->caption) : '' ) : + ( $status->rendered ?? $status->caption ); return [ '_v' => 1, @@ -34,7 +38,7 @@ class StatusStatelessTransformer extends Fractal\TransformerAbstract 'in_reply_to_id' => $status->in_reply_to_id ? (string) $status->in_reply_to_id : null, 'in_reply_to_account_id' => $status->in_reply_to_profile_id ? (string) $status->in_reply_to_profile_id : null, 'reblog' => $status->reblog_of_id ? StatusService::get($status->reblog_of_id, false) : null, - 'content' => $status->rendered ?? $status->caption, + 'content' => $rendered, 'content_text' => $status->caption, 'created_at' => str_replace('+00:00', 'Z', $status->created_at->format(DATE_RFC3339_EXTENDED)), 'emojis' => CustomEmoji::scan($status->caption), diff --git a/app/Transformer/Api/StatusTransformer.php b/app/Transformer/Api/StatusTransformer.php index d04b025f8..22a840ce0 100644 --- a/app/Transformer/Api/StatusTransformer.php +++ b/app/Transformer/Api/StatusTransformer.php @@ -19,6 +19,7 @@ use Illuminate\Support\Str; use App\Services\PollService; use App\Models\CustomEmoji; use App\Services\BookmarkService; +use App\Util\Lexer\Autolink; class StatusTransformer extends Fractal\TransformerAbstract { @@ -27,6 +28,9 @@ class StatusTransformer extends Fractal\TransformerAbstract $pid = request()->user()->profile_id; $taggedPeople = MediaTagService::get($status->id); $poll = $status->type === 'poll' ? PollService::get($status->id, $pid) : null; + $rendered = config('exp.autolink') ? + ( $status->caption ? Autolink::create()->autolink($status->caption) : '' ) : + ( $status->rendered ?? $status->caption ); return [ '_v' => 1, @@ -37,7 +41,7 @@ class StatusTransformer extends Fractal\TransformerAbstract 'in_reply_to_id' => (string) $status->in_reply_to_id, 'in_reply_to_account_id' => (string) $status->in_reply_to_profile_id, 'reblog' => $status->reblog_of_id ? StatusService::get($status->reblog_of_id) : null, - 'content' => $status->rendered ?? $status->caption, + 'content' => $rendered, 'content_text' => $status->caption, 'created_at' => str_replace('+00:00', 'Z', $status->created_at->format(DATE_RFC3339_EXTENDED)), 'emojis' => CustomEmoji::scan($status->caption), diff --git a/app/Util/ActivityPub/Helpers.php b/app/Util/ActivityPub/Helpers.php index 7f47a8fea..1304f0811 100644 --- a/app/Util/ActivityPub/Helpers.php +++ b/app/Util/ActivityPub/Helpers.php @@ -108,7 +108,10 @@ class Helpers { 'string', Rule::in($mimeTypes) ], - '*.name' => 'sometimes|nullable|string' + '*.name' => 'sometimes|nullable|string', + '*.blurhash' => 'sometimes|nullable|string|min:6|max:164', + '*.width' => 'sometimes|nullable|integer|min:1|max:5000', + '*.height' => 'sometimes|nullable|integer|min:1|max:5000', ])->passes(); return $valid; @@ -276,7 +279,7 @@ class Helpers { } if(is_array($val)) { - return !empty($val) ? $val[0] : null; + return !empty($val) ? head($val) : null; } return null; @@ -684,6 +687,8 @@ class Helpers { $blurhash = isset($media['blurhash']) ? $media['blurhash'] : null; $license = isset($media['license']) ? License::nameToId($media['license']) : null; $caption = isset($media['name']) ? Purify::clean($media['name']) : null; + $width = isset($media['width']) ? $media['width'] : false; + $height = isset($media['height']) ? $media['height'] : false; $media = new Media(); $media->blurhash = $blurhash; @@ -695,6 +700,12 @@ class Helpers { $media->remote_url = $url; $media->caption = $caption; $media->order = $key + 1; + if($width) { + $media->width = $width; + } + if($height) { + $media->height = $height; + } if($license) { $media->license = $license; } @@ -785,11 +796,12 @@ class Helpers { 'inbox_url' => $res['inbox'], 'outbox_url' => isset($res['outbox']) ? $res['outbox'] : null, 'public_key' => $res['publicKey']['publicKeyPem'], + 'indexable' => isset($res['indexable']) && is_bool($res['indexable']) ? $res['indexable'] : false, ] ); if( $profile->last_fetched_at == null || - $profile->last_fetched_at->lt(now()->subHours(24)) + $profile->last_fetched_at->lt(now()->subMonths(3)) ) { RemoteAvatarFetch::dispatch($profile); } diff --git a/app/Util/Lexer/Regex.php b/app/Util/Lexer/Regex.php index ecc468d05..f8d77c95f 100755 --- a/app/Util/Lexer/Regex.php +++ b/app/Util/Lexer/Regex.php @@ -162,7 +162,7 @@ abstract class Regex // look-ahead capture here and don't append $after when we return. $tmp['valid_mention_preceding_chars'] = '([^a-zA-Z0-9_!#\$%&*@＠\/]|^|(?:^|[^a-z0-9_+~.-])RT:?)'; - $re['valid_mentions_or_lists'] = '/'.$tmp['valid_mention_preceding_chars'].'(['.$tmp['at_signs'].'])([a-z0-9_\-.]{1,20})((\/[a-z][a-z0-9_\-]{0,24})?(?=(.*|$))(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i'; + $re['valid_mentions_or_lists'] = '/'.$tmp['valid_mention_preceding_chars'].'(['.$tmp['at_signs'].'])([\p{L}0-9_\-.]{1,20})((\/[a-z][a-z0-9_\-]{0,24})?(?=(.*|$))(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/iu'; $re['valid_reply'] = '/^(?:['.$tmp['spaces'].'])*['.$tmp['at_signs'].']([a-z0-9_\-.]{1,20})(?=(.*|$))/iu'; $re['end_mention_match'] = '/\A(?:['.$tmp['at_signs'].']|['.$tmp['latin_accents'].']|:\/\/)/iu'; diff --git a/app/Util/Media/Blurhash.php b/app/Util/Media/Blurhash.php index c0cca59b9..8e232ea17 100644 --- a/app/Util/Media/Blurhash.php +++ b/app/Util/Media/Blurhash.php @@ -44,6 +44,9 @@ class Blurhash { $pixels[] = $row; } + // Free the allocated GdImage object from memory: + imagedestroy($image); + $components_x = 4; $components_y = 4; $blurhash = BlurhashEngine::encode($pixels, $components_x, $components_y); @@ -53,4 +56,4 @@ class Blurhash { return $blurhash; } -} \ No newline at end of file +} diff --git a/app/Util/Site/Nodeinfo.php b/app/Util/Site/Nodeinfo.php index 166b6bc6a..0458299c5 100644 --- a/app/Util/Site/Nodeinfo.php +++ b/app/Util/Site/Nodeinfo.php @@ -2,85 +2,98 @@ namespace App\Util\Site; -use Cache; -use App\{Like, Profile, Status, User}; +use Illuminate\Support\Facades\Cache; +use App\Like; +use App\Profile; +use App\Status; +use App\User; use Illuminate\Support\Str; -class Nodeinfo { +class Nodeinfo +{ + public static function get() + { + $res = Cache::remember('api:nodeinfo', 900, function () { + $activeHalfYear = self::activeUsersHalfYear(); + $activeMonth = self::activeUsersMonthly(); - public static function get() - { - $res = Cache::remember('api:nodeinfo', 300, function () { - $activeHalfYear = Cache::remember('api:nodeinfo:ahy', 172800, function() { - return User::select('last_active_at') - ->where('last_active_at', '>', now()->subMonths(6)) - ->orWhere('created_at', '>', now()->subMonths(6)) - ->count(); - }); + $users = Cache::remember('api:nodeinfo:users', 43200, function() { + return User::count(); + }); - $activeMonth = Cache::remember('api:nodeinfo:am', 172800, function() { - return User::select('last_active_at') - ->where('last_active_at', '>', now()->subMonths(1)) - ->orWhere('created_at', '>', now()->subMonths(1)) - ->count(); - }); + $statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() { + return Status::whereLocal(true)->count(); + }); - $users = Cache::remember('api:nodeinfo:users', 43200, function() { - return User::count(); - }); + $features = [ 'features' => \App\Util\Site\Config::get()['features'] ]; - $statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() { - return Status::whereLocal(true)->count(); - }); + return [ + 'metadata' => [ + 'nodeName' => config_cache('app.name'), + 'software' => [ + 'homepage' => 'https://pixelfed.org', + 'repo' => 'https://github.com/pixelfed/pixelfed', + ], + 'config' => $features + ], + 'protocols' => [ + 'activitypub', + ], + 'services' => [ + 'inbound' => [], + 'outbound' => [], + ], + 'software' => [ + 'name' => 'pixelfed', + 'version' => config('pixelfed.version'), + ], + 'usage' => [ + 'localPosts' => (int) $statuses, + 'localComments' => 0, + 'users' => [ + 'total' => (int) $users, + 'activeHalfyear' => (int) $activeHalfYear, + 'activeMonth' => (int) $activeMonth, + ], + ], + 'version' => '2.0', + ]; + }); + $res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration'); + return $res; + } - $features = [ 'features' => \App\Util\Site\Config::get()['features'] ]; + public static function wellKnown() + { + return [ + 'links' => [ + [ + 'href' => config('pixelfed.nodeinfo.url'), + 'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0', + ], + ], + ]; + } - return [ - 'metadata' => [ - 'nodeName' => config_cache('app.name'), - 'software' => [ - 'homepage' => 'https://pixelfed.org', - 'repo' => 'https://github.com/pixelfed/pixelfed', - ], - 'config' => $features - ], - 'protocols' => [ - 'activitypub', - ], - 'services' => [ - 'inbound' => [], - 'outbound' => [], - ], - 'software' => [ - 'name' => 'pixelfed', - 'version' => config('pixelfed.version'), - ], - 'usage' => [ - 'localPosts' => (int) $statuses, - 'localComments' => 0, - 'users' => [ - 'total' => (int) $users, - 'activeHalfyear' => (int) $activeHalfYear, - 'activeMonth' => (int) $activeMonth, - ], - ], - 'version' => '2.0', - ]; - }); - $res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration'); - return $res; - } - - public static function wellKnown() - { - return [ - 'links' => [ - [ - 'href' => config('pixelfed.nodeinfo.url'), - 'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0', - ], - ], - ]; - } + public static function activeUsersMonthly() + { + return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function() { + return User::withTrashed() + ->select('last_active_at, updated_at') + ->where('updated_at', '>', now()->subWeeks(5)) + ->orWhere('last_active_at', '>', now()->subWeeks(5)) + ->count(); + }); + } + public static function activeUsersHalfYear() + { + return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function() { + return User::withTrashed() + ->select('last_active_at, updated_at') + ->where('last_active_at', '>', now()->subMonths(6)) + ->orWhere('updated_at', '>', now()->subMonths(6)) + ->count(); + }); + } } diff --git a/config/exp.php b/config/exp.php index 0ace5135b..e14463411 100644 --- a/config/exp.php +++ b/config/exp.php @@ -41,4 +41,6 @@ return [ // Post Update/Edits 'pue' => env('EXP_PUE', true), + + 'autolink' => env('EXP_AUTOLINK_V2', false), ]; diff --git a/config/filesystems.php b/config/filesystems.php index 6817d5e34..80e63ed99 100644 --- a/config/filesystems.php +++ b/config/filesystems.php @@ -79,6 +79,34 @@ return [ 'throw' => true, ], + 'alt-primary' => [ + 'enabled' => env('ALT_PRI_ENABLED', false), + 'driver' => 's3', + 'key' => env('ALT_PRI_AWS_ACCESS_KEY_ID'), + 'secret' => env('ALT_PRI_AWS_SECRET_ACCESS_KEY'), + 'region' => env('ALT_PRI_AWS_DEFAULT_REGION'), + 'bucket' => env('ALT_PRI_AWS_BUCKET'), + 'visibility' => 'public', + 'url' => env('ALT_PRI_AWS_URL'), + 'endpoint' => env('ALT_PRI_AWS_ENDPOINT'), + 'use_path_style_endpoint' => env('ALT_PRI_AWS_USE_PATH_STYLE_ENDPOINT', false), + 'throw' => true, + ], + + 'alt-secondary' => [ + 'enabled' => env('ALT_SEC_ENABLED', false), + 'driver' => 's3', + 'key' => env('ALT_SEC_AWS_ACCESS_KEY_ID'), + 'secret' => env('ALT_SEC_AWS_SECRET_ACCESS_KEY'), + 'region' => env('ALT_SEC_AWS_DEFAULT_REGION'), + 'bucket' => env('ALT_SEC_AWS_BUCKET'), + 'visibility' => 'public', + 'url' => env('ALT_SEC_AWS_URL'), + 'endpoint' => env('ALT_SEC_AWS_ENDPOINT'), + 'use_path_style_endpoint' => env('ALT_SEC_AWS_USE_PATH_STYLE_ENDPOINT', false), + 'throw' => true, + ], + 'spaces' => [ 'driver' => 's3', 'key' => env('DO_SPACES_KEY'), diff --git a/config/media.php b/config/media.php index b7d6e95cc..f550ff291 100644 --- a/config/media.php +++ b/config/media.php @@ -1,24 +1,26 @@ env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true), + 'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true), - 'exif' => [ - 'database' => env('MEDIA_EXIF_DATABASE', false), - ], + 'exif' => [ + 'database' => env('MEDIA_EXIF_DATABASE', false), + ], - 'storage' => [ - 'remote' => [ - /* - |-------------------------------------------------------------------------- - | Store remote media on cloud/S3 - |-------------------------------------------------------------------------- - | - | Set this to cache remote media on cloud/S3 filesystem drivers. - | Disabled by default. - | - */ - 'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false) - ], - ] + 'storage' => [ + 'remote' => [ + /* + |-------------------------------------------------------------------------- + | Store remote media on cloud/S3 + |-------------------------------------------------------------------------- + | + | Set this to cache remote media on cloud/S3 filesystem drivers. + | Disabled by default. + | + */ + 'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false), + + 'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false), + ], + ] ]; diff --git a/config/pixelfed.php b/config/pixelfed.php index 18e78b21d..fcdb1a4b7 100644 --- a/config/pixelfed.php +++ b/config/pixelfed.php @@ -23,7 +23,7 @@ return [ | This value is the version of your Pixelfed instance. | */ - 'version' => '0.11.8', + 'version' => '0.11.9', /* |-------------------------------------------------------------------------- diff --git a/config/remote-auth.php b/config/remote-auth.php index 3f85b9d40..182bb99a7 100644 --- a/config/remote-auth.php +++ b/config/remote-auth.php @@ -3,6 +3,7 @@ return [ 'mastodon' => [ 'enabled' => env('PF_LOGIN_WITH_MASTODON_ENABLED', false), + 'ignore_closed_state' => env('PF_LOGIN_WITH_MASTODON_ENABLED_SKIP_CLOSED', false), 'contraints' => [ /* diff --git a/database/migrations/2021_08_04_095125_create_groups_table.php b/database/migrations/2021_08_04_095125_create_groups_table.php new file mode 100644 index 000000000..29c63f73e --- /dev/null +++ b/database/migrations/2021_08_04_095125_create_groups_table.php @@ -0,0 +1,42 @@ +bigInteger('id')->unsigned()->primary(); + $table->bigInteger('profile_id')->unsigned()->nullable()->index(); + $table->string('status')->nullable()->index(); + $table->string('name')->nullable(); + $table->text('description')->nullable(); + $table->text('rules')->nullable(); + $table->boolean('local')->default(true)->index(); + $table->string('remote_url')->nullable(); + $table->string('inbox_url')->nullable(); + $table->boolean('is_private')->default(false); + $table->boolean('local_only')->default(false); + $table->json('metadata')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('groups'); + } +} diff --git a/database/migrations/2021_08_04_095143_create_group_members_table.php b/database/migrations/2021_08_04_095143_create_group_members_table.php new file mode 100644 index 000000000..33df26229 --- /dev/null +++ b/database/migrations/2021_08_04_095143_create_group_members_table.php @@ -0,0 +1,40 @@ +id(); + $table->bigInteger('group_id')->unsigned()->index(); + $table->bigInteger('profile_id')->unsigned()->index(); + $table->string('role')->default('member')->index(); + $table->boolean('local_group')->default(false)->index(); + $table->boolean('local_profile')->default(false)->index(); + $table->boolean('join_request')->default(false)->index(); + $table->timestamp('approved_at')->nullable(); + $table->timestamp('rejected_at')->nullable(); + $table->unique(['group_id', 'profile_id']); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('group_members'); + } +} diff --git a/database/migrations/2021_08_04_095238_create_group_posts_table.php b/database/migrations/2021_08_04_095238_create_group_posts_table.php new file mode 100644 index 000000000..a5e637d8e --- /dev/null +++ b/database/migrations/2021_08_04_095238_create_group_posts_table.php @@ -0,0 +1,42 @@ +bigInteger('id')->unsigned()->primary(); + $table->bigInteger('group_id')->unsigned()->index(); + $table->bigInteger('profile_id')->unsigned()->nullable()->index(); + $table->string('type')->nullable()->index(); + $table->bigInteger('status_id')->unsigned()->unique(); + $table->string('remote_url')->unique()->nullable()->index(); + $table->bigInteger('reply_child_id')->unsigned()->nullable(); + $table->bigInteger('in_reply_to_id')->unsigned()->nullable(); + $table->bigInteger('reblog_of_id')->unsigned()->nullable(); + $table->unsignedInteger('reply_count')->nullable(); + $table->string('status')->nullable()->index(); + $table->json('metadata')->nullable(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('group_posts'); + } +} diff --git a/database/migrations/2021_08_16_072457_create_group_invitations_table.php b/database/migrations/2021_08_16_072457_create_group_invitations_table.php new file mode 100644 index 000000000..aa13db23a --- /dev/null +++ b/database/migrations/2021_08_16_072457_create_group_invitations_table.php @@ -0,0 +1,38 @@ +bigIncrements('id'); + $table->bigInteger('group_id')->unsigned()->index(); + $table->bigInteger('from_profile_id')->unsigned()->index(); + $table->bigInteger('to_profile_id')->unsigned()->index(); + $table->string('role')->nullable(); + $table->boolean('to_local')->default(true)->index(); + $table->boolean('from_local')->default(true)->index(); + $table->unique(['group_id', 'to_profile_id']); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::dropIfExists('group_invitations'); + } +} diff --git a/database/migrations/2023_08_25_050021_add_indexable_column_to_profiles_table.php b/database/migrations/2023_08_25_050021_add_indexable_column_to_profiles_table.php new file mode 100644 index 000000000..f735366bd --- /dev/null +++ b/database/migrations/2023_08_25_050021_add_indexable_column_to_profiles_table.php @@ -0,0 +1,28 @@ +boolean('indexable')->default(false)->index(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('profiles', function (Blueprint $table) { + $table->dropColumn('indexable'); + }); + } +}; diff --git a/database/migrations/2023_09_12_044900_create_admin_shadow_filters_table.php b/database/migrations/2023_09_12_044900_create_admin_shadow_filters_table.php new file mode 100644 index 000000000..6b62f32c2 --- /dev/null +++ b/database/migrations/2023_09_12_044900_create_admin_shadow_filters_table.php @@ -0,0 +1,47 @@ +id(); + $table->unsignedBigInteger('admin_id')->nullable(); + $table->morphs('item'); + $table->boolean('is_local')->default(true)->index(); + $table->text('note')->nullable(); + $table->boolean('active')->default(false)->index(); + $table->json('history')->nullable(); + $table->json('ruleset')->nullable(); + $table->boolean('prevent_ap_fanout')->default(false)->index(); + $table->boolean('prevent_new_dms')->default(false)->index(); + $table->boolean('ignore_reports')->default(false)->index(); + $table->boolean('ignore_mentions')->default(false)->index(); + $table->boolean('ignore_links')->default(false)->index(); + $table->boolean('ignore_hashtags')->default(false)->index(); + $table->boolean('hide_from_public_feeds')->default(false)->index(); + $table->boolean('hide_from_tag_feeds')->default(false)->index(); + $table->boolean('hide_embeds')->default(false)->index(); + $table->boolean('hide_from_story_carousel')->default(false)->index(); + $table->boolean('hide_from_search_autocomplete')->default(false)->index(); + $table->boolean('hide_from_search')->default(false)->index(); + $table->boolean('requires_login')->default(false)->index(); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('admin_shadow_filters'); + } +}; diff --git a/resources/views/admin/asf/create.blade.php b/resources/views/admin/asf/create.blade.php new file mode 100644 index 000000000..8fc88e4bd --- /dev/null +++ b/resources/views/admin/asf/create.blade.php @@ -0,0 +1,64 @@ +@extends('admin.partial.template-full') + +@section('section') +
+
+
+
+
+

New Shadow Filters

+

Creating a new admin shadow filter

+
+
+
+
+
+
+
+
+
+ @if ($errors->any()) +
+
    + @foreach ($errors->all() as $error) +
  • {{ $error }}
    • + @endforeach +
+
+ @endif +
+
+ @csrf +
+ + +
+ +

Filters

+
+
+
+ + +
+
+ {{--
--}} +
+ +
+ + +
+
+ + +
+
+ +
+
+
+
+
+
+@endsection diff --git a/resources/views/admin/asf/edit.blade.php b/resources/views/admin/asf/edit.blade.php new file mode 100644 index 000000000..6d7a633f0 --- /dev/null +++ b/resources/views/admin/asf/edit.blade.php @@ -0,0 +1,64 @@ +@extends('admin.partial.template-full') + +@section('section') +
+
+
+
+
+

Edit Shadow Filters

+

Editing shadow filters

+
+
+
+
+
+
+
+
+
+ @if ($errors->any()) +
+
    + @foreach ($errors->all() as $error) +
  • {{ $error }}
    • + @endforeach +
+
+ @endif +
+
+ @csrf +
+ + +
+ +

Filters

+
+
+
+ hide_from_public_feeds ? 'checked=""' : '' !!}> + +
+
+ {{--
--}} +
+ +
+ + +
+
+ active ? 'checked=""' : ''}}> + +
+
+ +
+
+
+
+
+
+@endsection diff --git a/resources/views/admin/asf/home.blade.php b/resources/views/admin/asf/home.blade.php new file mode 100644 index 000000000..4fbb7730f --- /dev/null +++ b/resources/views/admin/asf/home.blade.php @@ -0,0 +1,81 @@ +@extends('admin.partial.template-full') + +@section('section') +
+
+
+
+
+

Admin Shadow Filters

+

Manage shadow filters across Accounts, Hashtags, Feeds and Stories

+
+
+
+
+
+
+
+
+
+ +
+ +
+
+ +
+
+
+ +
+ + + + + + + + + + + + @foreach($filters as $filter) + + + + + + + + @endforeach + +
IDUsernameHide FeedsActiveCreated
{{ $filter->id }} +
+ + +

+ @{{ $filter->account()['acct'] }} +

+
+ 		{{ $filter->hide_from_public_feeds ? '✅' : ''}}{{ $filter->active ? '✅' : ''}}{{ $filter->created_at->diffForHumans() }}
+ +
+ {{ $filters->links() }} +
+
+
+
+@endsection diff --git a/resources/views/auth/login.blade.php b/resources/views/auth/login.blade.php index 12b6b6f52..3403cd6b3 100644 --- a/resources/views/auth/login.blade.php +++ b/resources/views/auth/login.blade.php @@ -74,7 +74,10 @@ - @if(config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled')) + @if( + (config_cache('pixelfed.open_registration') && config('remote-auth.mastodon.enabled')) || + (config('remote-auth.mastodon.ignore_closed_state') && config('remote-auth.mastodon.enabled')) + )
@csrf diff --git a/resources/views/profile/embed.blade.php b/resources/views/profile/embed.blade.php index 0050a90e2..cc6097e3a 100644 --- a/resources/views/profile/embed.blade.php +++ b/resources/views/profile/embed.blade.php @@ -1,7 +1,7 @@ - + @@ -16,8 +16,8 @@ - - + + + } +
-
-
-
- - - {{$profile['username']}} - +
+
+ -
-
-
-
-

-

Posts

-
-
-

-

Followers

-
-
-

Follow

-
-
-
-
-
-

- View More Posts -

+
+
+
+
+

+

Posts

+
+

+

Followers

- - - - - - + + + + + - + }); + + diff --git a/resources/views/settings/partial/sidebar.blade.php b/resources/views/settings/partial/sidebar.blade.php index b4acf8c9b..a3837066a 100644 --- a/resources/views/settings/partial/sidebar.blade.php +++ b/resources/views/settings/partial/sidebar.blade.php @@ -72,6 +72,8 @@ @media only screen and (min-width: 768px) { border-right: 1px solid #dee2e6 !important } + height: 100%; + flex-grow: 1; } @endpush diff --git a/resources/views/settings/privacy.blade.php b/resources/views/settings/privacy.blade.php index 78ead55ee..57f83c664 100644 --- a/resources/views/settings/privacy.blade.php +++ b/resources/views/settings/privacy.blade.php @@ -28,9 +28,17 @@
crawlable ? 'checked=""':''}} {{$settings->is_private ? 'disabled=""':''}}> -

When your account is visible to search engines, your information can be crawled and stored by search engines.

+

When your account is visible to search engines, your information can be crawled and stored by search engines. {!! $settings->is_private ? 'Not available when your account is private' : ''!!}

+
+ +
+ indexable ? 'checked=""':''}} {{$settings->is_private ? 'disabled=""':''}}> + +

Your public posts may appear in search results on Pixelfed and Mastodon. People who have interacted with your posts may be able to search them regardless. {!! $settings->is_private ? 'Not available when your account is private' : ''!!}

@@ -39,7 +47,7 @@ -

When this option is enabled, your profile is included in the Directory. Only public profiles are eligible.

+

When this option is enabled, your profile is included in the Directory. Only public profiles are eligible. {!! $settings->is_private ? 'Not available when your account is private' : ''!!}

@@ -97,10 +105,10 @@

Enable your profile atom feed. Only public profiles are eligible.

@if($settings->show_atom)

- - {{ $profile->permalink('.atom') }} - - + + {{ $profile->permalink('.atom') }} + +

@endif
diff --git a/routes/api.php b/routes/api.php index f305f277c..23abfc323 100644 --- a/routes/api.php +++ b/routes/api.php @@ -316,6 +316,7 @@ Route::group(['prefix' => 'api'], function() use($middleware) { Route::post('seen', 'Stories\StoryApiV1Controller@viewed')->middleware($middleware); Route::post('self-expire/{id}', 'Stories\StoryApiV1Controller@delete')->middleware($middleware); Route::post('comment', 'Stories\StoryApiV1Controller@comment')->middleware($middleware); + Route::get('viewers', 'Stories\StoryApiV1Controller@viewers')->middleware($middleware); }); }); }); diff --git a/routes/web.php b/routes/web.php index bb091fce5..b823b8729 100644 --- a/routes/web.php +++ b/routes/web.php @@ -96,6 +96,13 @@ Route::domain(config('pixelfed.domain.admin'))->prefix('i/admin')->group(functio Route::get('autospam/home', 'AdminController@autospamHome')->name('admin.autospam'); + Route::redirect('asf/', 'asf/home'); + Route::get('asf/home', 'AdminShadowFilterController@home'); + Route::get('asf/create', 'AdminShadowFilterController@create'); + Route::get('asf/edit/{id}', 'AdminShadowFilterController@edit'); + Route::post('asf/edit/{id}', 'AdminShadowFilterController@storeEdit'); + Route::post('asf/create', 'AdminShadowFilterController@store'); + Route::prefix('api')->group(function() { Route::get('stats', 'AdminController@getStats'); Route::get('accounts', 'AdminController@getAccounts'); diff --git a/tests/Unit/ActivityPubTagObjectTest.php b/tests/Unit/ActivityPubTagObjectTest.php new file mode 100644 index 000000000..f402ff0da --- /dev/null +++ b/tests/Unit/ActivityPubTagObjectTest.php @@ -0,0 +1,133 @@ + [ + "href" => "https://gotosocial.example.org/users/GotosocialUser", + "name" => "@GotosocialUser@gotosocial.example.org", + "type" => "Mention" + ] + ]; + + if(isset($res['tag']['type'], $res['tag']['name'])) { + $res['tag'] = [$res['tag']]; + } + + $tags = collect($res['tag']) + ->filter(function($tag) { + return $tag && + $tag['type'] == 'Mention' && + isset($tag['href']) && + substr($tag['href'], 0, 8) === 'https://'; + }); + $this->assertTrue($tags->count() === 1); + } + + public function test_pixelfed_hashtags(): void + { + $res = [ + "tag" => [ + [ + "type" => "Mention", + "href" => "https://pixelfed.social/dansup", + "name" => "@dansup@pixelfed.social" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/dogsofpixelfed", + "name" => "#dogsOfPixelFed" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/doggo", + "name" => "#doggo" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/dog", + "name" => "#dog" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/drake", + "name" => "#drake" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/blacklab", + "name" => "#blacklab" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/iconic", + "name" => "#Iconic" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/majestic", + "name" => "#majestic" + ] + ] + ]; + + if(isset($res['tag']['type'], $res['tag']['name'])) { + $res['tag'] = [$res['tag']]; + } + + $tags = collect($res['tag']) + ->filter(function($tag) { + return $tag && + $tag['type'] == 'Hashtag' && + isset($tag['href']) && + substr($tag['href'], 0, 8) === 'https://'; + }); + $this->assertTrue($tags->count() === 7); + } + + + public function test_pixelfed_mentions(): void + { + $res = [ + "tag" => [ + [ + "type" => "Mention", + "href" => "https://pixelfed.social/dansup", + "name" => "@dansup@pixelfed.social" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/dogsofpixelfed", + "name" => "#dogsOfPixelFed" + ], + [ + "type" => "Hashtag", + "href" => "https://pixelfed.social/discover/tags/doggo", + "name" => "#doggo" + ], + ] + ]; + + if(isset($res['tag']['type'], $res['tag']['name'])) { + $res['tag'] = [$res['tag']]; + } + + $tags = collect($res['tag']) + ->filter(function($tag) { + return $tag && + $tag['type'] == 'Mention' && + isset($tag['href']) && + substr($tag['href'], 0, 8) === 'https://'; + }); + $this->assertTrue($tags->count() === 1); + } +} diff --git a/tests/Unit/Lexer/UsernameTest.php b/tests/Unit/Lexer/UsernameTest.php index 0d21b6e00..64875df7e 100644 --- a/tests/Unit/Lexer/UsernameTest.php +++ b/tests/Unit/Lexer/UsernameTest.php @@ -175,4 +175,67 @@ class UsernameTest extends TestCase $this->assertEquals($expectedEntity, $entities); } + /** @test * */ + public function germanUmlatsAutolink() + { + $mentions = "@März and @königin and @Glück"; + $autolink = Autolink::create()->autolink($mentions); + + $expectedAutolink = '@März and @königin and @Glück'; + $this->assertEquals($expectedAutolink, $autolink); + } + + /** @test * */ + public function germanUmlatsExtractor() + { + $mentions = "@März and @königin and @Glück"; + $entities = Extractor::create()->extract($mentions); + + $expectedEntity = [ + "hashtags" => [], + "urls" => [], + "mentions" => [ + "märz", + "königin", + "glück", + ], + "replyto" => null, + "hashtags_with_indices" => [], + "urls_with_indices" => [], + "mentions_with_indices" => [ + [ + "screen_name" => "März", + "indices" => [ + 0, + 5, + ], + ], + [ + "screen_name" => "königin", + "indices" => [ + 10, + 18, + ], + ], + [ + "screen_name" => "Glück", + "indices" => [ + 23, + 29, + ], + ], + ], + ]; + $this->assertEquals($expectedEntity, $entities); + } + + /** @test * */ + public function germanUmlatsWebfingerAutolink() + { + $mentions = "hello @märz@example.org!"; + $autolink = Autolink::create()->autolink($mentions); + + $expectedAutolink = 'hello @märz@example.org!'; + $this->assertEquals($expectedAutolink, $autolink); + } }