-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor jobs to use threads & update vulnerable dependencies (#44)
* Update dependencies (vulnerabilities) * Refactor jobs code, remove bullmq * fix worker thread & remove bullmq * lint * fix lint * quick rename variables for clarity
- Loading branch information
Showing
20 changed files
with
1,402 additions
and
2,681 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| * text=auto eol=lf |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,25 +1,13 @@ | ||
| import { Controller, Get } from '@nestjs/common'; | ||
| import { ApiOperation } from '@nestjs/swagger'; | ||
|
|
||
| import { QueuesService } from './jobs/queues.service'; | ||
|
|
||
| @Controller() | ||
| export class AppController { | ||
| constructor(private readonly queuesService: QueuesService) {} | ||
| @Get() | ||
| @ApiOperation({ | ||
| summary: 'Hello World', | ||
| }) | ||
| public getHello(): string { | ||
| return 'Hello World!'; | ||
| } | ||
|
|
||
| //For test purposes | ||
| @Get('process') | ||
| @ApiOperation({ | ||
| summary: 'Trigger the queue to process jobs', | ||
| }) | ||
| public async triggerProcessPrograms() { | ||
| await this.queuesService.processJobs(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| import { Controller, Get } from '@nestjs/common'; | ||
|
|
||
| import { JobsService } from './jobs.service'; | ||
|
|
||
| @Controller('jobs') | ||
| export class JobsController { | ||
| constructor(private readonly jobsService: JobsService) {} | ||
|
|
||
| @Get('run-workers') | ||
| public async runWorkers() { | ||
| await this.jobsService.processJobs(); | ||
| return { status: 'Jobs have been triggered' }; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| import { Module } from '@nestjs/common'; | ||
| import { ScheduleModule } from '@nestjs/schedule'; | ||
|
|
||
| import { CheminotModule } from '../common/api-helper/cheminot/cheminot.module'; | ||
| import { EtsModule } from '../common/api-helper/ets/ets.module'; | ||
| import { CourseModule } from '../course/course.module'; | ||
| import { ProgramModule } from '../program/program.module'; | ||
| import { ProgramCourseModule } from '../program-course/program-course.module'; | ||
| import { JobsController } from './jobs.controller'; | ||
| import { JobsService } from './jobs.service'; | ||
| import { CoursesJobService } from './workers/courses.worker'; | ||
| import { ProgramsJobService } from './workers/programs.worker'; | ||
|
|
||
| @Module({ | ||
| imports: [ | ||
| ScheduleModule.forRoot(), | ||
| EtsModule, | ||
| ProgramModule, | ||
| CourseModule, | ||
| ProgramCourseModule, | ||
| CheminotModule, | ||
| ], | ||
| providers: [JobsService, ProgramsJobService, CoursesJobService], | ||
| controllers: process.env.NODE_ENV === 'development' ? [JobsController] : [], | ||
| }) | ||
| export class JobsModule {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| import { Injectable, Logger } from '@nestjs/common'; | ||
| import { Cron, CronExpression } from '@nestjs/schedule'; | ||
| import { join } from 'path'; | ||
| import { isMainThread, Worker } from 'worker_threads'; | ||
|
|
||
| @Injectable() | ||
| export class JobsService { | ||
| private readonly logger = new Logger(JobsService.name); | ||
|
|
||
| private checkMainThread() { | ||
| this.logger.debug( | ||
| 'Are we on the main thread?', | ||
| isMainThread ? 'Yes' : 'No', | ||
| ); | ||
| } | ||
|
|
||
| private runWorker<T>(serviceName: string, methodName: string): Promise<T> { | ||
| return new Promise<T>((resolve, reject) => { | ||
| const workerScript = join(__dirname, 'workers', 'jobRunner.worker.js'); | ||
| const workerData = { serviceName, methodName }; | ||
| const worker = new Worker(workerScript, { workerData }); | ||
|
|
||
| worker.on('message', (message) => { | ||
| this.logger.verbose('Worker message:', message); | ||
| resolve(message); | ||
| }); | ||
|
|
||
| worker.on('error', (error) => { | ||
| this.logger.error('Worker error:', error); | ||
|
|
||
| const rejectionError = | ||
| error instanceof Error ? error : new Error(String(error)); | ||
| reject(rejectionError); | ||
| }); | ||
|
|
||
| worker.on('exit', (code) => { | ||
| if (code !== 0) { | ||
| this.logger.error(`Worker stopped with exit code ${code}`); | ||
| reject(new Error(`Worker stopped with exit code ${code}`)); | ||
| } | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| @Cron(CronExpression.EVERY_1ST_DAY_OF_MONTH_AT_MIDNIGHT) | ||
| public async processJobs(): Promise<void> { | ||
| this.logger.log('Starting job processing...'); | ||
| this.checkMainThread(); | ||
|
|
||
| try { | ||
| await Promise.all([ | ||
| this.runWorker('ProgramsJobService', 'processPrograms'), | ||
| this.runWorker('CoursesJobService', 'processCourses'), | ||
| this.runWorker( | ||
| 'CoursesJobService', | ||
| 'syncCourseDetailsWithCheminotData', | ||
| ), | ||
| ]); | ||
| this.logger.log('All jobs completed successfully!'); | ||
| } catch (error) { | ||
| this.logger.error('Job processing error:', error); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.