change exlude comments to include comments

dodona-edu · Nov 19, 2024 · 2d8328c · 2d8328c
1 parent e137cb5
commit 2d8328c
Show file tree

Hide file tree

Showing 9 changed files with 39 additions and 121 deletions.
diff --git a/cli/src/cli/commands/run.ts b/cli/src/cli/commands/run.ts
@@ -167,9 +167,9 @@ export function runCommand(program: Command): Command {
       Options.defaultKgramsInWindow
     )
     .option(
-      "-e, --exclude-comments",
+      "-ic, --include-comments",
       Utils.indent(
-        "Exclude the comments during the tokenization process."
+        "Include the comments during the tokenization process."
       )
     )
     .action(async (locations, options) => run(locations, { ...options , ...program.opts() }));
@@ -212,7 +212,7 @@ export async function run(locations: string[], options: RunOptions): Promise<voi
       limitResults: options.limitResults,
       sortBy: options.sortBy,
       fragmentSortBy: options.fragmentSortBy,
-      excludeComments: options.excludeComments
+      includeComments: options.includeComments
     });
     const report = await dolos.analyzePaths(locations, options.ignore);
 

diff --git a/lib/src/lib/dolos.ts b/lib/src/lib/dolos.ts
@@ -38,7 +38,7 @@ export class Dolos {
         this.language = this.languagePicker.detectLanguage(files);
         this.languageDetected = true;
       }
-      this.tokenizer = await this.language.createTokenizer({ excludeComments: this.options.excludeComments });
+      this.tokenizer = await this.language.createTokenizer({ includeComments: this.options.includeComments });
       this.index = new FingerprintIndex(this.options.kgramLength, this.options.kgramsInWindow, this.options.kgramData);
     }
     const warnings = [];

diff --git a/lib/src/lib/language.ts b/lib/src/lib/language.ts
@@ -25,7 +25,7 @@ export abstract class Language {
     }
   }
 
-  public abstract createTokenizer(options: TokenizerOptions): Promise<Tokenizer>;
+  public abstract createTokenizer(options?: TokenizerOptions): Promise<Tokenizer>;
 }
 
 export class ProgrammingLanguage extends Language {
@@ -54,7 +54,7 @@ export class ProgrammingLanguage extends Language {
     return this.languageModule;
   }
 
-  async createTokenizer(options: TokenizerOptions): Promise<Tokenizer> {
+  async createTokenizer(options?: TokenizerOptions): Promise<Tokenizer> {
     const { CodeTokenizer } = await import ("./tokenizer/codeTokenizer.js");
     await this.loadLanguageModule();
     return new CodeTokenizer(this, options);
@@ -86,12 +86,12 @@ export class CustomTokenizerLanguage extends Language {
   constructor(
     readonly name: string,
     readonly extensions: string[],
-    readonly customTokenizer: ((self: Language, options: TokenizerOptions) => Promise<Tokenizer>)
+    readonly customTokenizer: ((self: Language, options?: TokenizerOptions) => Promise<Tokenizer>)
   ) {
     super(name, extensions);
   }
 
-  public async createTokenizer(options: TokenizerOptions): Promise<Tokenizer> {
+  public async createTokenizer(options?: TokenizerOptions): Promise<Tokenizer> {
     return await this.customTokenizer(this, options);
   }
 }

diff --git a/lib/src/lib/options.ts b/lib/src/lib/options.ts
@@ -11,7 +11,7 @@ export interface DolosOptions {
   sortBy: string | null;
   fragmentSortBy: string | null;
   kgramData: boolean;
-  excludeComments: boolean;
+  includeComments: boolean;
 }
 
 export type CustomOptions = Partial<DolosOptions>;
@@ -88,8 +88,8 @@ export class Options implements DolosOptions {
     return this.custom.kgramData == true;
   }
 
-  get excludeComments(): boolean {
-    return this.custom.excludeComments === true;
+  get includeComments(): boolean {
+    return this.custom.includeComments === true;
   }
 
   get limitResults(): number | null {
@@ -159,7 +159,7 @@ export class Options implements DolosOptions {
       sortBy: this.sortBy,
       fragmentSortBy: this.fragmentSortBy,
       kgramData: this.kgramData,
-      excludeComments: this.excludeComments,
+      includeComments: this.includeComments,
     };
   }
 

diff --git a/lib/src/lib/tokenizer/codeTokenizer.ts b/lib/src/lib/tokenizer/codeTokenizer.ts
@@ -14,7 +14,7 @@ export class CodeTokenizer extends Tokenizer {
    * @param language The language to use for this tokenizer.
    * @param options
    */
-  constructor(language: ProgrammingLanguage, options: TokenizerOptions) {
+  constructor(language: ProgrammingLanguage, options?: TokenizerOptions) {
     super(language, options);
     this.parser = new Parser();
     this.parser.setLanguage(language.getLanguageModule());
@@ -63,7 +63,7 @@ export class CodeTokenizer extends Tokenizer {
     );
 
     const isComment = node.type.includes("comment");
-    if (!this.options.excludeComments || !isComment) {
+    if (!isComment || this.options.includeComments) {
       tokens.push(this.newToken("(", location));
       tokens.push(this.newToken(node.type, location));
     }
@@ -78,7 +78,7 @@ export class CodeTokenizer extends Tokenizer {
       }
     }
 
-    if (!this.options.excludeComments || !isComment) {
+    if (!isComment || this.options.includeComments) {
       tokens.push(this.newToken(")", location));
     }
 

diff --git a/lib/src/lib/tokenizer/tokenizer.ts b/lib/src/lib/tokenizer/tokenizer.ts
@@ -6,16 +6,18 @@ export interface Token {
   location: Region;
 }
 
-export type TokenizerOptions = {
-  excludeComments: boolean;
-}
+export type TokenizerOptions = Partial<{
+  includeComments: boolean;
+}>
 
 export abstract class Tokenizer {
 
-  protected options: TokenizerOptions;
+  protected options: TokenizerOptions = {};
 
-  constructor(public readonly language: Language, options: TokenizerOptions) {
-    this.options = options;
+  constructor(public readonly language: Language, options?: TokenizerOptions) {
+    if (options !== undefined) {
+      this.options = options;
+    }
   }