Add course upsertion job (#35)

* Add common folder

* Add program resource

* Fix eslint rule (complexity) & fix dependencies versions

* add course, cours-instance

* add session resource

* Add prisma, scraper, program files (modules and services)

* Fix prisma & add scraper files & eslint migration

* chore: address sonarlint warning (if statement)

* Rollback eslint flat to v8 & remove entity & format imports

* add public to classes

* cleanup

* add ets api hekper files

* add ets-program logic & (WIP) upsert programs

* Remove unnecessary dtos

* (wip 🚔) Add controllers & fix course routes

* fix tests

* Fix ProgramType relation

* Add coursePrerequisite controller & service

* Move scraper folder into /common

* Move pdf folder into /common

* fix http & add prisma to app module

* fix tests (wip) & sessions route

* fix tests

* rename

* Add schema image generator

* add methods

* fix

* change "program to programType" relation

* fix ci test

* miaow miaow

* test test

* only run eslint on src folder

* Rollback to ".test.ts" only

* pdf small test

* fix logging

* fix PR & remove scraper folder

* update enum trimester

* remove file

* fix param

* Add swagger

* fix program creation parameters & some route parameters

* fix axios vulnerability

* add queues for programs and courses processing

* add bull dashboard

* add planification tests

* yummy

* Extract cheminements.txt file text

* (wip) parse programs and courses

* (wip) parsing of courses partially done

* add cours hors-programme & remove commented lines from parsing

* remove console.log

* uppercase

* add support of alternative courses (choix)

* change id to code

* Add directory parameter to FileUtil

* pdfOutputPath

* remove json

* program upsert job creating  programs and program types to db

* fix progress% job

* change course code to null

* log verbose & fix course data format

* a

* test etsCourseService

* refactor upsertCourses() to use findMany() for better performance

* change prerequisite table relations

* coverage folder to root

* fix prerequisite service

* cleanup course service

* add program course resource

* add type column to ProgramCourse table

* cleanup program & courses cheminot helper

* (wip) programCourse creation almost done

* programCourse upsertion done

* Refactor logger initialization to use readonly modifier

* Add error handling

* fix sonar issues

* more fix

* add redis env variables

* fix type name

.env.example

@@ -5,3 +5,6 @@ PORT=3000
 DATABASE_URL="postgresql://postgres@localhost:5432/planifetsDB?schema=public"
 
 LOG_LEVELS="log,error,warn,debug,verbose"
+
+REDIS_HOST="localhost"
+REDIS_PORT=6379

.vscode/settings.json

@@ -3,7 +3,7 @@
   "[typescript]": {
     "editor.tabSize": 2,
     "editor.formatOnSave": true,
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
+    "editor.defaultFormatter": "dbaeumer.vscode-eslint"
   },
   "files.eol": "\n",
   "typescript.preferences.importModuleSpecifier": "relative",

package.json

@@ -104,7 +104,7 @@
     "collectCoverageFrom": [
       "**/*.(t|j)s"
     ],
-    "coverageDirectory": "../coverage",
+    "coverageDirectory": "./coverage",
     "testEnvironment": "node"
   }
 }

prisma/migrations/20240830020859_update_course_table_credits_attribute_to_nullable/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "Course" ALTER COLUMN "code" DROP NOT NULL,
+ALTER COLUMN "credits" DROP NOT NULL;

prisma/migrations/20240830021032_update_course_table_code_attribute_to_not_null/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Made the column `code` on table `Course` required. This step will fail if there are existing NULL values in that column.
+
+*/
+-- AlterTable
+ALTER TABLE "Course" ALTER COLUMN "code" SET NOT NULL;

prisma/migrations/20240830022905_update_course_table_add_cycle_attribute/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Course" ADD COLUMN     "cycle" INTEGER;

prisma/migrations/20240831045028_course_prerequisite_table_change_relation_to_one_to_many_program_course_table/migration.sql

@@ -0,0 +1,24 @@
+/*
+  Warnings:
+
+  - The primary key for the `CoursePrerequisite` table will be changed. If it partially fails, the table could be left without primary key constraint.
+  - Added the required column `programId` to the `CoursePrerequisite` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- DropForeignKey
+ALTER TABLE "CoursePrerequisite" DROP CONSTRAINT "CoursePrerequisite_courseId_fkey";
+
+-- DropForeignKey
+ALTER TABLE "CoursePrerequisite" DROP CONSTRAINT "CoursePrerequisite_prerequisiteId_fkey";
+
+-- AlterTable
+ALTER TABLE "CoursePrerequisite" DROP CONSTRAINT "CoursePrerequisite_pkey",
+ADD COLUMN     "programId" INTEGER NOT NULL,
+ADD COLUMN     "unstructuredPrerequisite" TEXT,
+ADD CONSTRAINT "CoursePrerequisite_pkey" PRIMARY KEY ("courseId", "programId", "prerequisiteId");
+
+-- AddForeignKey
+ALTER TABLE "CoursePrerequisite" ADD CONSTRAINT "CoursePrerequisite_courseId_programId_fkey" FOREIGN KEY ("courseId", "programId") REFERENCES "ProgramCourse"("courseId", "programId") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "CoursePrerequisite" ADD CONSTRAINT "CoursePrerequisite_prerequisiteId_programId_fkey" FOREIGN KEY ("prerequisiteId", "programId") REFERENCES "ProgramCourse"("courseId", "programId") ON DELETE RESTRICT ON UPDATE CASCADE;

prisma/migrations/20240902020425_update_program_course_table_add_type_attribute/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "ProgramCourse" ADD COLUMN     "type" TEXT,
+ALTER COLUMN "typicalSessionIndex" DROP NOT NULL;

prisma/schema.prisma

@@ -51,40 +51,44 @@ model Course {
   code            String           @unique
   title           String
   description     String
-  credits         Int
+  credits         Int?
+  cycle           Int?
   programs        ProgramCourse[]
   courseInstances CourseInstance[]
 
-  prerequisites  CoursePrerequisite[] @relation("CourseToPrerequisite")
-  prerequisiteOf CoursePrerequisite[] @relation("PrerequisiteToCourse")
-
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
   @@index([code])
 }
 
 model CoursePrerequisite {
-  courseId       Int
-  prerequisiteId Int
+  courseId                 Int
+  prerequisiteId           Int
+  unstructuredPrerequisite String?
+  programId                Int
 
-  course       Course @relation("CourseToPrerequisite", fields: [courseId], references: [id])
-  prerequisite Course @relation("PrerequisiteToCourse", fields: [prerequisiteId], references: [id])
+  programCourse ProgramCourse @relation("CourseToPrerequisites", fields: [courseId, programId], references: [courseId, programId])
+  prerequisite  ProgramCourse @relation("PrerequisiteToCourse", fields: [prerequisiteId, programId], references: [courseId, programId])
 
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 
-  @@id([courseId, prerequisiteId])
+  @@id([courseId, programId, prerequisiteId])
 }
 
 model ProgramCourse {
   courseId            Int
   programId           Int
-  typicalSessionIndex Int
+  type                String?
+  typicalSessionIndex Int?
 
   course  Course  @relation(fields: [courseId], references: [id])
   program Program @relation(fields: [programId], references: [id])
 
+  prerequisites        CoursePrerequisite[] @relation("CourseToPrerequisites")
+  prerequisiteToCourse CoursePrerequisite[] @relation("PrerequisiteToCourse")
+
   createdAt DateTime @default(now())
   updatedAt DateTime @updatedAt
 

src/app.module.ts

@@ -17,6 +17,7 @@ import { QueuesEnum } from './jobs/queues.enum';
 import { QueuesService } from './jobs/queues.service';
 import { PrismaModule } from './prisma/prisma.module';
 import { ProgramModule } from './program/program.module';
+import { ProgramCourseModule } from './program-course/program-course.module';
 import { SessionModule } from './session/session.module';
 
 @Module({
@@ -47,6 +48,7 @@ import { SessionModule } from './session/session.module';
     CoursePrerequisiteModule,
     SessionModule,
     ProgramModule,
+    ProgramCourseModule,
   ],
   providers: [ProgramsProcessor, CoursesProcessor, QueuesService],
   controllers: [AppController],

src/common/api-helper/cheminot/Course.ts

@@ -4,7 +4,8 @@ export class Course {
   public static readonly COURSE_LINE_PARTS_COUNT = 11;
   public static readonly INTERNSHIP_LINE_PARTS_COUNT = 12;
 
-  private static courseCodeValidationPipe = new CourseCodeValidationPipe();
+  private static readonly courseCodeValidationPipe =
+    new CourseCodeValidationPipe();
 
   constructor(
     public type: string,

src/common/api-helper/cheminot/Program.ts

@@ -1,11 +1,11 @@
 import { Course } from './Course';
 
 export class Program {
-  private horsProgramme: string[] = [];
+  private readonly horsProgramme: string[] = [];
 
   constructor(
-    private code: number,
-    private courses: Course[],
+    public code: string,
+    public courses: Course[],
   ) {}
 
   public static isProgramLine(line: string): boolean {
@@ -20,7 +20,7 @@ export class Program {
       return null;
     }
 
-    const code = parseInt(parts[1], 10);
+    const code = parts[1];
     return new Program(code, []);
   }
 

src/common/api-helper/cheminot/cheminot.controller.ts

@@ -23,9 +23,6 @@ export class CheminotController {
     summary: 'Parse the programs and courses from the cheminements.txt file',
   })
   public async parseProgramsAndCoursesFromCheminotTxtFile() {
-    await this.cheminotService.loadPrograms();
-    const data = this.cheminotService.getPrograms();
-
-    return data;
+    return this.cheminotService.parseProgramsAndCoursesCheminot();
   }
 }

src/common/api-helper/cheminot/cheminot.module.ts

@@ -7,5 +7,6 @@ import { FileExtractionService } from './file-extraction.service';
 @Module({
   controllers: [CheminotController],
   providers: [CheminotService, FileExtractionService],
+  exports: [CheminotService],
 })
 export class CheminotModule {}

src/common/api-helper/cheminot/cheminot.service.ts

@@ -6,10 +6,15 @@ import { Program } from './Program';
 
 @Injectable()
 export class CheminotService {
-  private programs: Program[] = [];
+  private readonly programs: Program[] = [];
 
   constructor(private readonly fileExtractionService: FileExtractionService) {}
 
+  public async parseProgramsAndCoursesCheminot() {
+    await this.loadPrograms();
+    return this.getPrograms();
+  }
+
   public async loadPrograms() {
     const fileContent =
       await this.fileExtractionService.extractCheminementsFile();

src/common/api-helper/ets/course/ets-course.controller.ts

@@ -3,8 +3,8 @@ import { ApiTags } from '@nestjs/swagger';
 
 import {
   EtsCourseService,
-  IEtsCourse,
-  IEtsCoursesData,
+  ICourseEtsAPI,
+  ICourseWithCredits,
 } from './ets-course.service';
 
 @ApiTags('ÉTS API')
@@ -13,12 +13,12 @@ export class EtsCourseController {
   constructor(private readonly etsCourseService: EtsCourseService) {}
 
   @Get()
-  public fetchAllCourses(): Promise<IEtsCoursesData[]> {
-    return this.etsCourseService.fetchAllCourses();
+  public fetchAllCourses(): Promise<ICourseWithCredits[]> {
+    return this.etsCourseService.fetchAllCoursesWithCredits();
   }
 
   @Get(':id')
-  public fetchCoursesById(@Param('id') id: string): Promise<IEtsCourse[]> {
+  public fetchCoursesById(@Param('id') id: string): Promise<ICourseEtsAPI[]> {
     if (!id) {
       throw new Error('The id parameter is required');
     }

src/common/api-helper/ets/course/ets-course.service.ts

@@ -6,50 +6,92 @@ import {
   ETS_API_GET_ALL_COURSES,
   ETS_API_GET_COURSES_BY_IDS,
 } from '../../../constants/url';
+import { extractNumberFromString } from '../../../utils/stringUtil';
 
-export interface IEtsCoursesData {
+export interface ICoursesEtsAPI {
   id: number;
   title: string;
+  description: string;
   code: string;
   cycle: string | null;
 }
 
-export interface IEtsCourse extends IEtsCoursesData {
-  credits: string;
+export interface ICourseEtsAPI {
+  id: number;
+  title: string;
+  code: string;
+  credits: number | null;
+}
+
+export interface ICourses {
+  id: number;
+  title: string;
+  description: string;
+  code: string;
+  cycle: number | null;
+}
+
+export interface ICourseWithCredits extends ICourses {
+  credits: number | null;
 }
 
 @Injectable()
 export class EtsCourseService {
   constructor(private readonly httpService: HttpService) {}
 
-  // Fetches all courses
-  public async fetchAllCourses(): Promise<IEtsCoursesData[]> {
+  public async fetchAllCoursesWithCredits(): Promise<ICourseWithCredits[]> {
+    const courses = await this.fetchAllCoursesWithoutCredits();
+
+    // Fetch credits for all courses by their IDs
+    const courseIds = courses.map((course) => course.id).join(',');
+    const coursesFetchedById = await this.fetchCoursesById(courseIds);
+
+    // Add credits to the courses
+    return courses.map((course) => {
+      const courseCreds = coursesFetchedById.find((cc) => cc.id === course.id);
+      return {
+        ...course,
+        credits: courseCreds?.credits ?? null,
+      };
+    });
+  }
+
+  //Fetches all courses without credits
+  public async fetchAllCoursesWithoutCredits(): Promise<ICourses[]> {
     const response = await firstValueFrom(
       this.httpService.get(ETS_API_GET_ALL_COURSES),
     );
-
     const courses = response.data.results;
-    return courses.map((course: IEtsCoursesData) => ({
+
+    if (!courses.length) {
+      throw new Error('No courses fetched.');
+    }
+
+    return courses.map((course: ICoursesEtsAPI) => ({
       id: course.id,
       title: course.title,
+      description: course.description,
       code: course.code,
-      cycle: course.cycle,
+      cycle: course.cycle ? extractNumberFromString(course.cycle) : null,
     }));
   }
 
   // Fetches one or more courses by their ids
   // The ids are passed as a string with comma-separated values, ex: "349682,349710"
-  public async fetchCoursesById(ids: string): Promise<IEtsCourse[]> {
+  public async fetchCoursesById(ids: string): Promise<ICourseEtsAPI[]> {
     const response = await firstValueFrom(
       this.httpService.get(`${ETS_API_GET_COURSES_BY_IDS}${ids}`),
     );
-
     const courses = response.data;
-    return courses.map((course: IEtsCourse) => ({
+
+    if (!courses.length) {
+      throw new Error('No courses fetched.');
+    }
+
+    return courses.map((course: ICourseEtsAPI) => ({
       id: course.id,
       title: course.title,
       code: course.code,
-      cycle: course.cycle,
       credits: course.credits,
     }));
   }

src/common/api-helper/ets/program/ets-program.service.ts

@@ -3,6 +3,7 @@ import { Injectable } from '@nestjs/common';
 import { firstValueFrom } from 'rxjs';
 
 import { ETS_API_GET_ALL_PROGRAMS } from '../../../constants/url';
+import { extractNumberFromString } from '../../../utils/stringUtil';
 
 interface IProgramEtsAPI {
   id: number;
@@ -48,7 +49,7 @@ export class EtsProgramService {
       (program: IProgramEtsAPI) => ({
         id: program.id,
         title: program.title,
-        cycle: this.extractCycleNumber(program.cycle),
+        cycle: extractNumberFromString(program.cycle),
         code: program.code,
         credits: program.credits,
         programTypes: {
@@ -60,10 +61,4 @@ export class EtsProgramService {
 
     return { types, programs };
   }
-
-  private extractCycleNumber(cycle: string): number {
-    const match = RegExp(/\d+/).exec(cycle);
-
-    return match ? parseInt(match[0], 10) : 0;
-  }
 }

src/common/utils/pdf/fileUtil.ts

@@ -4,7 +4,7 @@ import path from 'path';
 
 @Injectable()
 export class FileUtil {
-  private logger = new Logger(FileUtil.name);
+  private readonly logger = new Logger(FileUtil.name);
 
   public writeDataToFile<T>(
     data: T,

src/common/utils/pdf/parser/pdfParserUtil.ts

@@ -2,7 +2,7 @@ import { Logger } from '@nestjs/common';
 import PDFParser, { Output } from 'pdf2json';
 
 export class PdfParserUtil {
-  private static logger = new Logger(PdfParserUtil.name);
+  private static readonly logger = new Logger(PdfParserUtil.name);
 
   public static async parsePdfBuffer<T>(
     pdfBuffer: Buffer,

src/common/utils/stringUtil.ts

@@ -0,0 +1,5 @@
+export function extractNumberFromString(cycle: string): number {
+  const match = RegExp(/\d+/).exec(cycle);
+
+  return match ? parseInt(match[0], 10) : 0;
+}