From 12d31892c9f4779f7b9e87c9dd13b0279385251f Mon Sep 17 00:00:00 2001 From: pushrbx Date: Wed, 13 Nov 2024 16:58:51 +0000 Subject: [PATCH 1/4] fixed issues with incremental indexer command --- .../Commands/Indexer/IncrementalIndexer.php | 56 +++++++++++++++---- app/Providers/AppServiceProvider.php | 9 ++- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php index eb076ac0..98c040b3 100644 --- a/app/Console/Commands/Indexer/IncrementalIndexer.php +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -30,6 +30,23 @@ protected function promptForMissingArgumentsUsing(): array ]; } + private function sleep(int $milliseconds): void + { + $interval = 100; // check every 100 ms + $elapsed = 0; + + while ($elapsed < $milliseconds) + { + if ($this->cancelled) + { + return; + } + + usleep($interval * 1000); + $elapsed += $interval; + } + } + private function getExistingIds(string $mediaType): array { $existingIdsHash = ""; @@ -91,12 +108,12 @@ private function getFailedIdsToFetch(string $mediaType): array return json_decode(Storage::get("indexer/incremental/{$mediaType}_failed.json")); } - private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): void + private function fetchIds(string $mediaType, array $idsToFetch, int $delay, bool $resume): void { $index = 0; $success = []; $failedIds = []; - $idCount = count($idsToFetch); + if ($resume && Storage::exists("indexer/incremental/{$mediaType}_resume.save")) { $index = (int)Storage::get("indexer/incremental/{$mediaType}_resume.save"); @@ -104,6 +121,7 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v } $ids = array_merge($idsToFetch['sfw'], $idsToFetch['nsfw']); + $idCount = count($ids); if ($index > 0 && !isset($ids[$index])) { @@ -119,10 +137,11 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v { if ($this->cancelled) { + $this->info("Cancelling..."); return; } - $id = $ids[$index]; + $id = $ids[$i]; $url = env('APP_URL') . "/v4/$mediaType/$id"; $this->info("Indexing/Updating " . ($i + 1) . "/$idCount $url [MAL ID: $id]"); @@ -142,6 +161,16 @@ private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): v $this->warn("[SKIPPED] Failed to fetch $url"); $failedIds[] = $id; Storage::put("indexer/incremental/$mediaType.failed", json_encode($failedIds)); + continue; + } + finally + { + $this->sleep($delay * 1000); + if ($this->cancelled) + { + $this->info("Cancelling..."); + return; + } } $success[] = $id; @@ -168,18 +197,20 @@ public function handle(): int [ 'mediaType' => $this->argument('mediaType'), 'delay' => $this->option('delay'), - 'resume' => $this->option('resume') ?? false, - 'failed' => $this->option('failed') ?? false + 'resume' => $this->option('resume'), + 'failed' => $this->option('failed') ], [ - 'mediaType' => 'required|in:anime,manga', + 'mediaType' => 'required|array', + 'mediaType.*' => 'in:anime,manga', 'delay' => 'integer|min:1', - 'resume' => 'bool|prohibited_with:failed', - 'failed' => 'bool|prohibited_with:resume' + 'resume' => 'bool', + 'failed' => 'bool' ] ); - if ($validator->fails()) { + if ($validator->fails()) + { $this->error($validator->errors()->toJson()); return 1; } @@ -189,6 +220,7 @@ public function handle(): int $resume = $this->option('resume') ?? false; $onlyFailed = $this->option('failed') ?? false; + $delay = $this->option('delay') ?? 3; /** * @var $mediaTypes array @@ -211,16 +243,18 @@ public function handle(): int if ($this->cancelled) { - return 127; + $this->info("Cancelling..."); + return 0; } $idCount = count($idsToFetch); if ($idCount === 0) { + $this->info("No $mediaType entries to index"); continue; } - $this->fetchIds($mediaType, $idsToFetch, $resume); + $this->fetchIds($mediaType, $idsToFetch, $delay, $resume); } return 0; diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 4b8fb545..b74ad585 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -48,6 +48,7 @@ use App\Support\DefaultMediator; use App\Support\JikanConfig; use App\Support\JikanUnitOfWork; +use Illuminate\Console\Signals; use Illuminate\Contracts\Container\BindingResolutionException; use Illuminate\Http\JsonResponse; use Laravel\Lumen\Application; @@ -55,7 +56,6 @@ use Illuminate\Support\Env; use Illuminate\Support\ServiceProvider; use Illuminate\Support\Collection; -use Jikan\MyAnimeList\MalClient; use Laravel\Scout\Builder as ScoutBuilder; use Typesense\LaravelTypesense\Typesense; use App\Features; @@ -96,6 +96,13 @@ public function register(): void $this->app->singleton(\App\Services\TypesenseCollectionDescriptor::class); } $this->registerModelRepositories(); + + // lumen hack for signal handling in artisan commands + Signals::resolveAvailabilityUsing(function () { + return $this->app->runningInConsole() + && ! $this->app->runningUnitTests() + && extension_loaded('pcntl'); + }); } private function getSearchService(Repository $repository): SearchService From e213318869a8c2b665765aad58d64de00fe5aa7b Mon Sep 17 00:00:00 2001 From: pushrbx Date: Wed, 13 Nov 2024 19:27:05 +0000 Subject: [PATCH 2/4] changed exit code and refactored cancellable sleep function --- .../Commands/Indexer/IncrementalIndexer.php | 35 ++++++++----------- app/Support/helpers.php | 18 ++++++++++ 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php index 98c040b3..be50b376 100644 --- a/app/Console/Commands/Indexer/IncrementalIndexer.php +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -12,6 +12,7 @@ class IncrementalIndexer extends Command * @var bool */ private bool $cancelled = false; + private int $receivedSignal = 0; /** * The name and signature of the console command. @@ -30,23 +31,6 @@ protected function promptForMissingArgumentsUsing(): array ]; } - private function sleep(int $milliseconds): void - { - $interval = 100; // check every 100 ms - $elapsed = 0; - - while ($elapsed < $milliseconds) - { - if ($this->cancelled) - { - return; - } - - usleep($interval * 1000); - $elapsed += $interval; - } - } - private function getExistingIds(string $mediaType): array { $existingIdsHash = ""; @@ -71,10 +55,10 @@ private function getIdsToFetch(string $mediaType): array return []; } + $this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json..."); $newIdsRaw = file_get_contents("https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json"); $newIdsHash = sha1($newIdsRaw); - /** @noinspection PhpConditionAlreadyCheckedInspection */ if ($this->cancelled) { return []; @@ -165,7 +149,7 @@ private function fetchIds(string $mediaType, array $idsToFetch, int $delay, bool } finally { - $this->sleep($delay * 1000); + cancellable_sleep($delay * 1000, fn() => $this->cancelled); if ($this->cancelled) { $this->info("Cancelling..."); @@ -216,7 +200,12 @@ public function handle(): int } // we want to handle signals from the OS - $this->trap([SIGTERM, SIGQUIT, SIGINT], fn () => $this->cancelled = true); + $this->trap([SIGTERM, SIGQUIT, SIGINT], function (int $signal) { + $this->cancelled = true; + $this->receivedSignal = $signal; + }); + + $this->info("Info: IncrementalIndexer uses purarue/mal-id-cache fetch available MAL IDs and updates/indexes them\n\n"); $resume = $this->option('resume') ?? false; $onlyFailed = $this->option('failed') ?? false; @@ -244,7 +233,7 @@ public function handle(): int if ($this->cancelled) { $this->info("Cancelling..."); - return 0; + return 128 + $this->receivedSignal; } $idCount = count($idsToFetch); @@ -257,6 +246,10 @@ public function handle(): int $this->fetchIds($mediaType, $idsToFetch, $delay, $resume); } + if ($this->cancelled && $this->receivedSignal > 0) + { + return 128 + $this->receivedSignal; + } return 0; } } diff --git a/app/Support/helpers.php b/app/Support/helpers.php index df69542d..82ca907b 100644 --- a/app/Support/helpers.php +++ b/app/Support/helpers.php @@ -123,3 +123,21 @@ function ensureEnumPrimitiveValue(int|string|bool|float|null|\Spatie\Enum\Larave return $value; } } + +if (!function_exists("cancellable_sleep")) { + function cancellable_sleep(int $milliseconds, callable $isCancelled): void { + $interval = 100; // check every 100 ms + $elapsed = 0; + + while ($elapsed < $milliseconds) + { + if ($isCancelled()) + { + return; + } + + usleep($interval * 1000); + $elapsed += $interval; + } + } +} From 5e0d0fd5d8bb53aedd8c2015b62d9d5f43ee898c Mon Sep 17 00:00:00 2001 From: pushrbx Date: Wed, 13 Nov 2024 19:29:02 +0000 Subject: [PATCH 3/4] added PromptsForMissingInput contract to IncrementalIndexer --- app/Console/Commands/Indexer/IncrementalIndexer.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php index be50b376..490293d2 100644 --- a/app/Console/Commands/Indexer/IncrementalIndexer.php +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -3,10 +3,11 @@ namespace App\Console\Commands\Indexer; use Illuminate\Console\Command; +use Illuminate\Contracts\Console\PromptsForMissingInput; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Facades\Validator; -class IncrementalIndexer extends Command +class IncrementalIndexer extends Command implements PromptsForMissingInput { /** * @var bool From a318f243d6e38d50418382490425c92e89b285ab Mon Sep 17 00:00:00 2001 From: pushrbx Date: Wed, 13 Nov 2024 19:30:40 +0000 Subject: [PATCH 4/4] refactor --- app/Console/Commands/Indexer/IncrementalIndexer.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/Console/Commands/Indexer/IncrementalIndexer.php b/app/Console/Commands/Indexer/IncrementalIndexer.php index 490293d2..e3c1f904 100644 --- a/app/Console/Commands/Indexer/IncrementalIndexer.php +++ b/app/Console/Commands/Indexer/IncrementalIndexer.php @@ -247,10 +247,6 @@ public function handle(): int $this->fetchIds($mediaType, $idsToFetch, $delay, $resume); } - if ($this->cancelled && $this->receivedSignal > 0) - { - return 128 + $this->receivedSignal; - } - return 0; + return $this->cancelled && $this->receivedSignal > 0 ? 128 + $this->receivedSignal : 0; } }