diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php index eb076ac0..e3c1f904 100644 --- a/app/Console/Commands/Indexer/IncrementalIndexer.php +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -3,15 +3,17 @@ namespace App\Console\Commands\Indexer; use Illuminate\Console\Command; +use Illuminate\Contracts\Console\PromptsForMissingInput; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Validator; -class IncrementalIndexer extends Command +class IncrementalIndexer extends Command implements PromptsForMissingInput { /** * @var bool */ private bool $cancelled = false; + private int $receivedSignal = 0; /** * The name and signature of the console command. @@ -54,10 +56,10 @@ private function getIdsToFetch(string $mediaType): array return []; } + $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json..."); $newIdsRaw = file_get_contents("https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json"); $newIdsHash = sha1($newIdsRaw); - /** @noinspection PhpConditionAlreadyCheckedInspection */ if ($this->cancelled) { return []; @@ -91,12 +93,12 @@ private function getFailedIdsToFetch(string $mediaType): array return json_decode(Storage::get("indexer/incremental/{$mediaType}_failed.json")); } - private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): void + private function fetchIds(string $mediaType, array $idsToFetch, int $delay, bool $resume): void { $index = 0; $success = []; $failedIds = []; - $idCount = count($idsToFetch); + if ($resume && Storage::exists("indexer/incremental/{$mediaType}_resume.save")) { $index = (int)Storage::get("indexer/incremental/{$mediaType}_resume.save"); @@ -104,6 +106,7 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v } $ids = array_merge($idsToFetch['sfw'], $idsToFetch['nsfw']); + $idCount = count($ids); if ($index > 0 && !isset($ids[$index])) { @@ -119,10 +122,11 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v { if ($this->cancelled) { + $this->info("Cancelling..."); return; } - $id = $ids[$index]; + $id = $ids[$i]; $url = env('APP_URL') . "/v4/$mediaType/$id"; $this->info("Indexing/Updating " . ($i + 1) . "/$idCount $url [MAL ID: $id]"); @@ -142,6 +146,16 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v $this->warn("[SKIPPED] Failed to fetch $url"); $failedIds[] = $id; Storage::put("indexer/incremental/$mediaType.failed", json_encode($failedIds)); + continue; + } + finally + { + cancellable_sleep($delay * 1000, fn() => $this->cancelled); + if ($this->cancelled) + { + $this->info("Cancelling..."); + return; + } } $success[] = $id; @@ -168,27 +182,35 @@ public function handle(): int [ 'mediaType' => $this->argument('mediaType'), 'delay' => $this->option('delay'), - 'resume' => $this->option('resume') ?? false, - 'failed' => $this->option('failed') ?? false + 'resume' => $this->option('resume'), + 'failed' => $this->option('failed') ], [ - 'mediaType' => 'required|in:anime,manga', + 'mediaType' => 'required|array', + 'mediaType.*' => 'in:anime,manga', 'delay' => 'integer|min:1', - 'resume' => 'bool|prohibited_with:failed', - 'failed' => 'bool|prohibited_with:resume' + 'resume' => 'bool', + 'failed' => 'bool' ] ); - if ($validator->fails()) { + if ($validator->fails()) + { $this->error($validator->errors()->toJson()); return 1; } // we want to handle signals from the OS - $this->trap([SIGTERM, SIGQUIT, SIGINT], fn () => $this->cancelled = true); + $this->trap([SIGTERM, SIGQUIT, SIGINT], function (int $signal) { + $this->cancelled = true; + $this->receivedSignal = $signal; + }); + + $this->info("Info: IncrementalIndexer uses purarue/mal-id-cache fetch available MAL IDs and updates/indexes them\n\n"); $resume = $this->option('resume') ?? false; $onlyFailed = $this->option('failed') ?? false; + $delay = $this->option('delay') ?? 3; /** * @var $mediaTypes array @@ -211,18 +233,20 @@ public function handle(): int if ($this->cancelled) { - return 127; + $this->info("Cancelling..."); + return 128 + $this->receivedSignal; } $idCount = count($idsToFetch); if ($idCount === 0) { + $this->info("No $mediaType entries to index"); continue; } - $this->fetchIds($mediaType, $idsToFetch, $resume); + $this->fetchIds($mediaType, $idsToFetch, $delay, $resume); } - return 0; + return $this->cancelled && $this->receivedSignal > 0 ? 128 + $this->receivedSignal : 0; } } diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 4b8fb545..b74ad585 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -48,6 +48,7 @@ use App\Support\DefaultMediator; use App\Support\JikanConfig; use App\Support\JikanUnitOfWork; +use Illuminate\Console\Signals; use Illuminate\Contracts\Container\BindingResolutionException; use Illuminate\Http\JsonResponse; use Laravel\Lumen\Application; @@ -55,7 +56,6 @@ use Illuminate\Support\Env; use Illuminate\Support\ServiceProvider; use Illuminate\Support\Collection; -use Jikan\MyAnimeList\MalClient; use Laravel\Scout\Builder as ScoutBuilder; use Typesense\LaravelTypesense\Typesense; use App\Features; @@ -96,6 +96,13 @@ public function register(): void $this->app->singleton(\App\Services\TypesenseCollectionDescriptor::class); } $this->registerModelRepositories(); + + // lumen hack for signal handling in artisan commands + Signals::resolveAvailabilityUsing(function () { + return $this->app->runningInConsole() + && ! $this->app->runningUnitTests() + && extension_loaded('pcntl'); + }); } private function getSearchService(Repository $repository): SearchService diff --git a/app/Support/helpers.php b/app/Support/helpers.php index df69542d..82ca907b 100644 --- a/app/Support/helpers.php +++ b/app/Support/helpers.php @@ -123,3 +123,21 @@ function ensureEnumPrimitiveValue(int|string|bool|float|null|\Spatie\Enum\Larave return $value; } } + +if (!function_exists("cancellable_sleep")) { + function cancellable_sleep(int $milliseconds, callable $isCancelled): void { + $interval = 100; // check every 100 ms + $elapsed = 0; + + while ($elapsed < $milliseconds) + { + if ($isCancelled()) + { + return; + } + + usleep($interval * 1000); + $elapsed += $interval; + } + } +}