Merge pull request #3953 from pixelfed/staging

Update MediaS3GarbageCollector commmand, disable logging by default and optimize huge invocations

CHANGELOG.md

@@ -56,6 +56,7 @@
 - Update InboxPipeline, add inbox job queue and separate http sig validation from activity handling ([e6c1604d](https://github.com/pixelfed/pixelfed/commit/e6c1604d))
 - Update InboxPipeline, dispatch Follow/Accept Follow jobs to follow queue ([f62d2494](https://github.com/pixelfed/pixelfed/commit/f62d2494))
 - Add MediaS3GarbageCollector command to clear local media after uploaded to S3 disks after 12 hours ([b8c3f153](https://github.com/pixelfed/pixelfed/commit/b8c3f153))
+- Update MediaS3GarbageCollector command, disable logging by default and optimize huge invocations ([a14af93b](https://github.com/pixelfed/pixelfed/commit/a14af93b))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.4 (2022-10-04)](https://github.com/pixelfed/pixelfed/compare/v0.11.3...v0.11.4)

app/Console/Commands/MediaS3GarbageCollector.php

@@ -15,7 +15,7 @@ class MediaS3GarbageCollector extends Command
     *
     * @var string
     */
-    protected $signature = 'media:s3gc {--limit=200}';
+    protected $signature = 'media:s3gc {--limit=200} {--huge} {--log-errors}';
 
     /**
     * The console command description.
@@ -54,8 +54,23 @@ class MediaS3GarbageCollector extends Command
         }
 
         $limit = $this->option('limit');
+        $hugeMode = $this->option('huge');
+        $log = $this->option('log-errors');
+
+        if($limit > 2000 && !$hugeMode) {
+            $this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag');
+            return;
+        }
+
         $minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first()->id;
 
+        return $hugeMode ?
+            $this->hugeMode($minId, $limit, $log) :
+            $this->regularMode($minId, $limit, $log);
+    }
+
+    protected function regularMode($minId, $limit, $log)
+    {
         $gc = Media::whereRemoteMedia(false)
             ->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
             ->whereNot('version', '4')
@@ -84,8 +99,10 @@ class MediaS3GarbageCollector extends Command
                     $media->save();
                 }
             } else {
+                if($log) {
                     Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
                 }
+            }
             $bar->advance();
         }
         $bar->finish();
@@ -96,4 +113,42 @@ class MediaS3GarbageCollector extends Command
         }
         return 0;
     }
+
+    protected function hugeMode($minId, $limit, $log)
+    {
+        $cloudDisk = Storage::disk(config('filesystems.cloud'));
+        $localDisk = Storage::disk('local');
+
+        $bar = $this->output->createProgressBar($limit);
+        $bar->start();
+
+        Media::whereRemoteMedia(false)
+            ->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
+            ->whereNot('version', '4')
+            ->where('id', '<', $minId)
+            ->chunk(50, function($medias) use($cloudDisk, $localDisk, $bar, $log) {
+                foreach($medias as $media) {
+                    if($cloudDisk->exists($media->media_path)) {
+                        if( $localDisk->exists($media->media_path)) {
+                            $localDisk->delete($media->media_path);
+                            $media->version = 4;
+                            $media->save();
+                            $totalSize = $totalSize + $media->size;
+                        } else {
+                            $media->version = 4;
+                            $media->save();
+                        }
+                    } else {
+                        if($log) {
+                            Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
+                        }
+                    }
+                    $bar->advance();
+                }
+        });
+
+        $bar->finish();
+        $this->line(' ');
+        $this->info('Finished!');
+    }
 }