From c5aa50173281cf5b25b2074b06be481165c066b8 Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Wed, 31 Jan 2024 21:34:15 +0100 Subject: [PATCH 01/31] Made some changes to the code to support Docker --- Dockerfile | 13 ++++++++ package.json | 7 +++-- src/converter.ts | 78 ++++++++++++++++++++++++++++++++++++++++++++++++ src/index.ts | 64 ++------------------------------------- src/watcher.ts | 43 ++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 64 deletions(-) create mode 100644 Dockerfile create mode 100644 src/converter.ts create mode 100644 src/watcher.ts diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..62bd75e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM node:18-alpine3.19 + +WORKDIR /app + +COPY . . + +RUN npm install + +RUN mkdir /var/input +RUN mkdir /var/output + +ENTRYPOINT [ "npm" ] +CMD ["run", "watch"] \ No newline at end of file diff --git a/package.json b/package.json index f2d3d1d..a073b6c 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,9 @@ "name": "trading212-to-ghostfolio", "version": "1.0.0", "description": "Convert Trading 212 export to Ghostfolio import", - "main": "index.js", "scripts": { - "start": "nodemon" + "start": "ts-node ./src/index.ts", + "watch": "ts-node ./src/watcher.ts" }, "author": "Dick Wolff", "repository": { @@ -13,13 +13,14 @@ "license": "Apache-2.0", "devDependencies": { "@types/node": "^20.10.4", - "nodemon": "^3.0.2", "ts-node": "^10.9.1", "typescript": "^5.3.3" }, "dependencies": { "@types/cli-progress": "^3.11.5", + "chokidar": "^3.5.3", "cli-progress": "^3.12.0", + "closest-match": "^1.3.3", "cross-fetch": "^4.0.0", "csv-parse": "^5.5.2", "dayjs": "^1.11.10", diff --git a/src/converter.ts b/src/converter.ts new file mode 100644 index 0000000..ad29fad --- /dev/null +++ b/src/converter.ts @@ -0,0 +1,78 @@ +import path from "path"; +import * as fs from "fs"; +import { GhostfolioExport } from "./models/ghostfolioExport"; +import { DeGiroConverter } from "./converters/degiroConverter"; +import { SchwabConverter } from "./converters/schwabConverter"; +import { DeGiroConverterV2 } from "./converters/degiroConverterV2"; +import { AbstractConverter } from "./converters/abstractconverter"; +import { Trading212Converter } from "./converters/trading212Converter"; +import { SwissquoteConverter } from "./converters/swissquoteConverter"; +import { FinpensionConverter } from "./converters/finpensionConverter"; + +export function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath, completionCallback: CallableFunction, errorCallback: CallableFunction) { + + const converterTypeLc = converterType.toLocaleLowerCase(); + + // Determine convertor type. + const converter = createConverter(converterTypeLc); + + // Map the file to a Ghostfolio import. + converter.processFile(inputFilePath, (result: GhostfolioExport) => { + + console.log("[i] Processing complete, writing to file..") + + // Write result to file. + const outputFileName = path.join(outputFilePath, `ghostfolio-${converterTypeLc}.json`); + const fileContents = JSON.stringify(result); + fs.writeFileSync(outputFileName, fileContents, { encoding: "utf-8" }); + + console.log(`[i] Wrote data to '${outputFileName}.json'!`); + + completionCallback(); + }); +} + +function createConverter(converterType: string): AbstractConverter { + + let converter: AbstractConverter; + + switch (converterType) { + case "t212": + case "trading212": + console.log("[i] Processing file using Trading212 converter"); + converter = new Trading212Converter(); + break; + case "degiro": + console.log("[i] Processing file using DeGiro converter"); + console.log("[i] NOTE: There is a new version available of the DeGiro converter"); + console.log("[i] The new converter has multiple record parsing improvements and also supports platform fees."); + console.log("[i] The new converter is currently in beta and we're looking for your feedback!"); + console.log("[i] You can run the beta converter with the command 'npm run start degiro-v2'."); + converter = new DeGiroConverter(); + break; + case "degiro-v2": + console.log("[i] Processing file using DeGiro converter (V2 Beta)"); + console.log("[i] NOTE: You are running a converter that is currently in beta."); + console.log("[i] If you have any issues, please report them on GitHub. Many thanks!"); + converter = new DeGiroConverterV2(); + break; + case "fp": + case "finpension": + console.log("[i] Processing file using Finpension converter"); + converter = new FinpensionConverter(); + break; + case "sq": + case "swissquote": + console.log("[i] Processing file using Swissquote converter"); + converter = new SwissquoteConverter(); + break; + case "schwab": + console.log("[i] Processing file using Schwab converter"); + converter = new SchwabConverter(); + break; + default: + throw new Error(`Unknown converter '${process.argv[2].toLocaleLowerCase()}' provided`); + } + + return converter; +} diff --git a/src/index.ts b/src/index.ts index b64cf7d..b4f7c61 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,67 +1,9 @@ -import * as fs from "fs"; -import { GhostfolioExport } from "./models/ghostfolioExport"; -import { DeGiroConverter } from "./converters/degiroConverter"; -import { DeGiroConverterV2 } from "./converters/degiroConverterV2"; -import { AbstractConverter } from "./converters/abstractconverter"; -import { Trading212Converter } from "./converters/trading212Converter"; -import { SchwabConverter } from "./converters/schwabConverter"; -import { SwissquoteConverter } from "./converters/swissquoteConverter"; -import { FinpensionConverter } from "./converters/finpensionConverter"; +import { createAndRunConverter } from "./converter"; require("dotenv").config(); // Define import file path. const inputFile = process.env.INPUT_FILE; -let converter: AbstractConverter; - -// Determine convertor type. -switch (process.argv[2].toLocaleLowerCase()) { - case "t212": - case "trading212": - console.log("[i] Processing file using Trading212 converter"); - converter = new Trading212Converter(); - break; - case "degiro": - console.log("[i] Processing file using DeGiro converter"); - console.log("[i] NOTE: There is a new version available of the DeGiro converter"); - console.log("[i] The new converter has multiple record parsing improvements and also supports platform fees."); - console.log("[i] The new converter is currently in beta and we're looking for your feedback!"); - console.log("[i] You can run the beta converter with the command 'npm run start degiro-v2'."); - converter = new DeGiroConverter(); - break; - case "degiro-v2": - console.log("[i] Processing file using DeGiro converter (V2 Beta)"); - console.log("[i] NOTE: You are running a converter that is currently in beta."); - console.log("[i] If you have any issues, please report them on GitHub. Many thanks!"); - converter = new DeGiroConverterV2(); - break; - case "fp": - case "finpension": - console.log("[i] Processing file using Finpension converter"); - converter = new FinpensionConverter(); - break; - case "sq": - case "swissquote": - console.log("[i] Processing file using Swissquote converter"); - converter = new SwissquoteConverter(); - break; - case "schwab": - console.log("[i] Processing file using Schwab converter"); - converter = new SchwabConverter(); - break; - default: - throw new Error(`Unknown converter '${process.argv[2].toLocaleLowerCase()}' provided`); -} - -// Map the file to a Ghostfolio import. -converter.processFile(inputFile, (result: GhostfolioExport) => { - - console.log("[i] Processing complete, writing to file..") - - // Write result to file. - const fileContents = JSON.stringify(result); - fs.writeFileSync(`ghostfolio-${process.argv[2].toLocaleLowerCase()}.json`, fileContents, { encoding: "utf-8" }); - - console.log(`[i] Wrote data to 'ghostfolio-${process.argv[2].toLocaleLowerCase()}.json'!`); -}); +// Determine convertor type and run conversion. +createAndRunConverter(process.argv[2].toLocaleLowerCase(), inputFile, ".", () => { }, () => { }); diff --git a/src/watcher.ts b/src/watcher.ts new file mode 100644 index 0000000..9de1f31 --- /dev/null +++ b/src/watcher.ts @@ -0,0 +1,43 @@ +import * as fs from "fs"; +import chokidar from "chokidar"; +import * as matcher from "closest-match"; +import { createAndRunConverter } from "./converter"; + +// Define input and output. +const inputFolder = process.env.E2G_INPUT_FOLDER || "/var/input"; +const outputFolder = process.env.E2G_OUTPUT_FOLDER || "/var/output"; + +console.log(`[i] Watching ${inputFolder}..\n`); + +chokidar.watch(inputFolder, { usePolling: true }).on("add", path => { + + console.log(`[i] Found ${path}!`); + + const fileContents = fs.readFileSync(path, "utf-8"); + + const closestMatch = matcher.closestMatch(fileContents.split("\n")[0], [...headers.keys()]); + + let converterKey = closestMatch as string; + + // If multiple matches were found (type would not be 'string'), pick the first. + if (typeof closestMatch !== "string") { + converterKey = closestMatch[0]; + } + + const converter = headers.get(converterKey); + console.log(`[i] Determined the file type to be of kind '${converter}'.`); + + // Determine convertor type and run conversion. + createAndRunConverter(converter, path, outputFolder, () => { + + // After conversion was succesful, remove input file. + console.log(`[i] Finished converting ${path}, removing file..\n\n`); + fs.rmSync(path); + }, () => { }); + +}); + +// Prep header set. +const headers: Map = new Map(); +headers.set("Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id", "degiro"); +headers.set("Action,Time,ISIN,Ticker,Name,No. of shares,Price / share,Currency (Price / share),Exchange rate,Result,Currency (Result),Total,Currency (Total),Withholding tax,Currency (Withholding tax),Notes,ID,Currency conversion fee", "trading212"); From c87bd5f55f79625abfa1ec9709d73b641e6155b8 Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Wed, 31 Jan 2024 21:34:59 +0100 Subject: [PATCH 02/31] Fix folder names in Docker --- Dockerfile | 4 ++-- src/watcher.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 62bd75e..36cbff7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,8 @@ COPY . . RUN npm install -RUN mkdir /var/input -RUN mkdir /var/output +RUN mkdir /var/e2g-input +RUN mkdir /var/e2g-output ENTRYPOINT [ "npm" ] CMD ["run", "watch"] \ No newline at end of file diff --git a/src/watcher.ts b/src/watcher.ts index 9de1f31..a4592ba 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -4,8 +4,8 @@ import * as matcher from "closest-match"; import { createAndRunConverter } from "./converter"; // Define input and output. -const inputFolder = process.env.E2G_INPUT_FOLDER || "/var/input"; -const outputFolder = process.env.E2G_OUTPUT_FOLDER || "/var/output"; +const inputFolder = process.env.E2G_INPUT_FOLDER || "/var/e2g-input"; +const outputFolder = process.env.E2G_OUTPUT_FOLDER || "/var/e2g-output"; console.log(`[i] Watching ${inputFolder}..\n`); From 35b30ed9e89d7c43407401397c3e6be1da142691 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Wed, 31 Jan 2024 21:39:44 +0100 Subject: [PATCH 03/31] Create workflow to publish Docker image --- .github/workflows/docker.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..a8c4ff7 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,28 @@ +name: ci + +on: + push: + branches: + - 'feature/Dockerize-With-File-Watch' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + push: true + tags: dickwolff/export-to-ghostfolio:latest From 3100deebe119d7ce8be58e288923e1721d427425 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Wed, 31 Jan 2024 21:42:43 +0100 Subject: [PATCH 04/31] Focus Docker build only on src directory changes --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a8c4ff7..005ef6f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,5 +24,6 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: + context: "{{defaultContext}}:src" push: true tags: dickwolff/export-to-ghostfolio:latest From 40198e7adca79f777d83282bd597268047298263 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Wed, 31 Jan 2024 21:44:40 +0100 Subject: [PATCH 05/31] Specify Dockerfile --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 005ef6f..ae1d97b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,3 +27,4 @@ jobs: context: "{{defaultContext}}:src" push: true tags: dickwolff/export-to-ghostfolio:latest + file: "src/Dockerfile" From b043ad82f444d236519087e6cd00fcc089dca3f5 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Wed, 31 Jan 2024 21:46:00 +0100 Subject: [PATCH 06/31] Change contexrt --- .github/workflows/docker.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ae1d97b..a8c4ff7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -24,7 +24,5 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 with: - context: "{{defaultContext}}:src" push: true tags: dickwolff/export-to-ghostfolio:latest - file: "src/Dockerfile" From 8b0018a8fe215e21de29053e3bf0378f8b3cbd1e Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:08:14 +0100 Subject: [PATCH 07/31] Add semver --- .github/workflows/docker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a8c4ff7..d968846 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,9 @@ jobs: docker: runs-on: ubuntu-latest steps: + - name: GitTools + uses: GitTools/actions@v0.10.2 + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -25,4 +28,4 @@ jobs: uses: docker/build-push-action@v5 with: push: true - tags: dickwolff/export-to-ghostfolio:latest + tags: dickwolff/export-to-ghostfolio:latest ${{ env.GitVersion_FullSemVer }} From 30ebe1f4a76d393c02c73441260de90de0d52a86 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:11:45 +0100 Subject: [PATCH 08/31] Fix semver --- .github/workflows/docker.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d968846..988f1da 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,10 +8,18 @@ on: jobs: docker: runs-on: ubuntu-latest - steps: - - name: GitTools - uses: GitTools/actions@v0.10.2 - + steps: + - name: Install GitVersion + uses: gittools/actions/gitversion/setup@v0.10.2 + with: + versionSpec: '5.x' + + - name: Run GitVersion + uses: gittools/actions/gitversion/execute@v0.10.2 + #with: + #useConfigFile: true + #configFilePath: GitVersion.yml + - name: Set up QEMU uses: docker/setup-qemu-action@v3 From 3cf447b64d87dd69b60007b42403f5da878d9d85 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:13:48 +0100 Subject: [PATCH 09/31] Use GitVersion file --- .github/workflows/docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 988f1da..185c782 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,9 +16,9 @@ jobs: - name: Run GitVersion uses: gittools/actions/gitversion/execute@v0.10.2 - #with: - #useConfigFile: true - #configFilePath: GitVersion.yml + with: + useConfigFile: true + configFilePath: GitVersion.yml - name: Set up QEMU uses: docker/setup-qemu-action@v3 From 125b1e538179ab2dfcd2b91b6bcc76dee279d049 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:14:41 +0100 Subject: [PATCH 10/31] Add GitVersion.yml --- GitVersion.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 GitVersion.yml diff --git a/GitVersion.yml b/GitVersion.yml new file mode 100644 index 0000000..67cbfb1 --- /dev/null +++ b/GitVersion.yml @@ -0,0 +1,15 @@ +next-version: 0.3.0 +assembly-informational-format: "{NuGetVersion}" +mode: ContinuousDeployment +branches: + master: + regex: main + mode: ContinuousDelivery + tag: "" + increment: Patch + feature: + regex: ^feature?[/-] + mode: ContinuousDelivery + tag: "" + increment: Patch + source-branches: ["main"] From cdd0c351f993fbebf0a170d71b323031efee32be Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:17:59 +0100 Subject: [PATCH 11/31] Update docker.yml --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 185c782..ec332bb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,7 +18,7 @@ jobs: uses: gittools/actions/gitversion/execute@v0.10.2 with: useConfigFile: true - configFilePath: GitVersion.yml + configFilePath: './GitVersion.yml' - name: Set up QEMU uses: docker/setup-qemu-action@v3 From d51b6912e4cde187ab82c28bccb3b3c6d3cf04fa Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:20:38 +0100 Subject: [PATCH 12/31] Update docker.yml --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ec332bb..0364ba6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -18,7 +18,7 @@ jobs: uses: gittools/actions/gitversion/execute@v0.10.2 with: useConfigFile: true - configFilePath: './GitVersion.yml' + #configFilePath: './GitVersion.yml' - name: Set up QEMU uses: docker/setup-qemu-action@v3 From 17f9b0c17abd427461f110cc4a42d3abc6a712b2 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:24:49 +0100 Subject: [PATCH 13/31] Print directory --- .github/workflows/docker.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0364ba6..c71d0eb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -13,7 +13,10 @@ jobs: uses: gittools/actions/gitversion/setup@v0.10.2 with: versionSpec: '5.x' - + + - name: print + run: ls + - name: Run GitVersion uses: gittools/actions/gitversion/execute@v0.10.2 with: From dc2820f2b458bb2e8c7e98beba6bab888603fdf2 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:26:02 +0100 Subject: [PATCH 14/31] Update docker.yml --- .github/workflows/docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c71d0eb..2c80d0f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,7 +15,9 @@ jobs: versionSpec: '5.x' - name: print - run: ls + run: | + echo "aa" + ls - name: Run GitVersion uses: gittools/actions/gitversion/execute@v0.10.2 From 365209689e53518e370365c7d348dc317f7d3ffb Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:26:43 +0100 Subject: [PATCH 15/31] Update docker.yml --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2c80d0f..8c09385 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,6 +17,7 @@ jobs: - name: print run: | echo "aa" + cd ../ ls - name: Run GitVersion From 99a67bcfb7bf38f1b8340900b38d6c3e5a41d6d0 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:27:36 +0100 Subject: [PATCH 16/31] Add more cd --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8c09385..da57156 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: - name: print run: | echo "aa" - cd ../ + cd ../../ ls - name: Run GitVersion From b092e614c381cdecfd111d0739e92a6ab764fb81 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:28:41 +0100 Subject: [PATCH 17/31] Checkout --- .github/workflows/docker.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index da57156..61682d7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,11 @@ jobs: docker: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Install GitVersion uses: gittools/actions/gitversion/setup@v0.10.2 with: @@ -17,8 +22,8 @@ jobs: - name: print run: | echo "aa" - cd ../../ ls + echo "bb" - name: Run GitVersion uses: gittools/actions/gitversion/execute@v0.10.2 From ea63317b53721ef27b76cb8fb46559c1bbbe1ae0 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:31:57 +0100 Subject: [PATCH 18/31] Remove debug actions --- .github/workflows/docker.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 61682d7..3ce3f71 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,19 +17,12 @@ jobs: - name: Install GitVersion uses: gittools/actions/gitversion/setup@v0.10.2 with: - versionSpec: '5.x' - - - name: print - run: | - echo "aa" - ls - echo "bb" + versionSpec: '5.x' - name: Run GitVersion uses: gittools/actions/gitversion/execute@v0.10.2 with: useConfigFile: true - #configFilePath: './GitVersion.yml' - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -47,4 +40,4 @@ jobs: uses: docker/build-push-action@v5 with: push: true - tags: dickwolff/export-to-ghostfolio:latest ${{ env.GitVersion_FullSemVer }} + tags: dickwolff/export-to-ghostfolio:latest ${{ env.GitVersion_MajorMinorPatch }} From 6888f19e9c02b70ccb369f869441f8ae0c88eabb Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:34:29 +0100 Subject: [PATCH 19/31] No push when PR build, fix tags --- .github/workflows/docker.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3ce3f71..b8cc112 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -38,6 +38,8 @@ jobs: - name: Build and push uses: docker/build-push-action@v5 - with: - push: true - tags: dickwolff/export-to-ghostfolio:latest ${{ env.GitVersion_MajorMinorPatch }} + with: + push: ${{ github.event_name != 'pull_request' }} + tags: | + dickwolff/export-to-ghostfolio:latest + dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }} From 75097d4f5e1d2a1af0077351f68988c4cbf3dde3 Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:43:54 +0100 Subject: [PATCH 20/31] Conditional tag if feature branch --- .github/workflows/docker.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b8cc112..4fd7850 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -36,10 +36,22 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set GitVersion tag if main mranch + run: | + echo "DOCKER_TAG=dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}" >> $GITHUB_ENV + if: ${{ github.event_name != 'pull_request' }} + + - name: Set GitVersion tag if feature branch + run: | + echo "DOCKER_TAG=dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}-beta" >> $GITHUB_ENV + if: ${{ github.event_name == 'pull_request' }} + - name: Build and push uses: docker/build-push-action@v5 + env: + GITVERSION_TAG: ${{ github.event_name != 'pull_request' }} with: push: ${{ github.event_name != 'pull_request' }} tags: | dickwolff/export-to-ghostfolio:latest - dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }} + ${{ env.DOCKER_TAG }} From d1b05f24b1a38adf78cc7a4d699701f8ba894d3f Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:49:25 +0100 Subject: [PATCH 21/31] Check for main branch --- .github/workflows/docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4fd7850..33ce1a8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,14 +37,14 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set GitVersion tag if main mranch + if: ${{ github.ref == 'refs/heads/main' }} run: | echo "DOCKER_TAG=dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}" >> $GITHUB_ENV - if: ${{ github.event_name != 'pull_request' }} - + - name: Set GitVersion tag if feature branch + if: ${{ github.ref != 'refs/heads/main' }} run: | - echo "DOCKER_TAG=dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}-beta" >> $GITHUB_ENV - if: ${{ github.event_name == 'pull_request' }} + echo "DOCKER_TAG=dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}-beta" >> $GITHUB_ENV - name: Build and push uses: docker/build-push-action@v5 From 54349d2311d311bd765cf6bda6866fa5fc18430e Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:55:15 +0100 Subject: [PATCH 22/31] Alleen docker push op main --- .github/workflows/docker.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 33ce1a8..cb8ffba 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,7 +3,7 @@ name: ci on: push: branches: - - 'feature/Dockerize-With-File-Watch' + - 'main' jobs: docker: @@ -36,22 +36,10 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set GitVersion tag if main mranch - if: ${{ github.ref == 'refs/heads/main' }} - run: | - echo "DOCKER_TAG=dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}" >> $GITHUB_ENV - - - name: Set GitVersion tag if feature branch - if: ${{ github.ref != 'refs/heads/main' }} - run: | - echo "DOCKER_TAG=dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }}-beta" >> $GITHUB_ENV - - name: Build and push uses: docker/build-push-action@v5 - env: - GITVERSION_TAG: ${{ github.event_name != 'pull_request' }} with: push: ${{ github.event_name != 'pull_request' }} tags: | dickwolff/export-to-ghostfolio:latest - ${{ env.DOCKER_TAG }} + dickwolff/export-to-ghostfolio:${{ env.GitVersion_MajorMinorPatch }} From 8b54a114c0e687c6c92068de15f2b621c17d5ba5 Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Thu, 1 Feb 2024 10:01:25 +0100 Subject: [PATCH 23/31] Add other converters to file watcher --- src/watcher.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/watcher.ts b/src/watcher.ts index a4592ba..3179967 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -39,5 +39,8 @@ chokidar.watch(inputFolder, { usePolling: true }).on("add", path => { // Prep header set. const headers: Map = new Map(); -headers.set("Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id", "degiro"); -headers.set("Action,Time,ISIN,Ticker,Name,No. of shares,Price / share,Currency (Price / share),Exchange rate,Result,Currency (Result),Total,Currency (Total),Withholding tax,Currency (Withholding tax),Notes,ID,Currency conversion fee", "trading212"); +headers.set(`Datum,Tijd,Valutadatum,Product,ISIN,Omschrijving,FX,Mutatie,,Saldo,,Order Id`, "degiro"); +headers.set(`Date;Category;"Asset Name";ISIN;"Number of Shares";"Asset Currency";"Currency Rate";"Asset Price in CHF";"Cash Flow";Balance`, "finpension"); +headers.set(`Date,Action,Symbol,Description,Quantity,Price,Fees & Comm,Amount`, "schwab"); +headers.set(`Date;Order #;Transaction;Symbol;Name;ISIN;Quantity;Unit price;Costs;Accrued Interest;Net Amount;Balance;Currency`, "swissquote"); +headers.set(`Action,Time,ISIN,Ticker,Name,No. of shares,Price / share,Currency (Price / share),Exchange rate,Result,Currency (Result),Total,Currency (Total),Withholding tax,Currency (Withholding tax),Notes,ID,Currency conversion fee`, "trading212"); From c6b83ea48df4fd69a6143473252a4efab0f341ea Mon Sep 17 00:00:00 2001 From: Dick Wolff <5620002+dickwolff@users.noreply.github.com> Date: Thu, 1 Feb 2024 10:06:04 +0100 Subject: [PATCH 24/31] Rename ci --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cb8ffba..026e4d0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: ci +name: docker on: push: From 8959c63a2a333514569ae601cf0fc8d78f27cdd0 Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Thu, 1 Feb 2024 11:35:58 +0100 Subject: [PATCH 25/31] Fix converter issues --- src/converter.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/converter.ts b/src/converter.ts index ad29fad..163fb48 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -9,7 +9,7 @@ import { Trading212Converter } from "./converters/trading212Converter"; import { SwissquoteConverter } from "./converters/swissquoteConverter"; import { FinpensionConverter } from "./converters/finpensionConverter"; -export function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath, completionCallback: CallableFunction, errorCallback: CallableFunction) { +export function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) { const converterTypeLc = converterType.toLocaleLowerCase(); @@ -17,7 +17,7 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri const converter = createConverter(converterTypeLc); // Map the file to a Ghostfolio import. - converter.processFile(inputFilePath, (result: GhostfolioExport) => { + converter.readAndProcessFile(inputFilePath, (result: GhostfolioExport) => { console.log("[i] Processing complete, writing to file..") @@ -29,7 +29,7 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri console.log(`[i] Wrote data to '${outputFileName}.json'!`); completionCallback(); - }); + }, (error) => errorCallback(error)); } function createConverter(converterType: string): AbstractConverter { @@ -71,7 +71,7 @@ function createConverter(converterType: string): AbstractConverter { converter = new SchwabConverter(); break; default: - throw new Error(`Unknown converter '${process.argv[2].toLocaleLowerCase()}' provided`); + throw new Error(`Unknown converter '${converterType}' provided`); } return converter; From 3f413f8f6dd6d8b13f97b9ae906e9266beab7b19 Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Thu, 1 Feb 2024 11:38:21 +0100 Subject: [PATCH 26/31] Add schwab converter test --- src/converters/degiroConverterV2.test.ts | 2 +- src/converters/schwabConverter.test.ts | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 src/converters/schwabConverter.test.ts diff --git a/src/converters/degiroConverterV2.test.ts b/src/converters/degiroConverterV2.test.ts index a034063..2a647f9 100644 --- a/src/converters/degiroConverterV2.test.ts +++ b/src/converters/degiroConverterV2.test.ts @@ -1,6 +1,6 @@ import { DeGiroConverterV2 } from "./degiroConverterV2"; -describe("degiroConverter", () => { +describe("degiroConverterV2", () => { it("should construct", () => { diff --git a/src/converters/schwabConverter.test.ts b/src/converters/schwabConverter.test.ts new file mode 100644 index 0000000..79a45dd --- /dev/null +++ b/src/converters/schwabConverter.test.ts @@ -0,0 +1,13 @@ +import { SchwabConverter } from "./schwabConverter"; + +describe("SchwabConverter", () => { + + it("should construct", () => { + + // Act + const sut = new SchwabConverter(); + + // Asssert + expect(sut).toBeTruthy(); + }); +}); From 12bf8fc0e88fee62ad53fb7ff807a6f81983d75f Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Thu, 1 Feb 2024 11:45:56 +0100 Subject: [PATCH 27/31] Move file to output folder when an error ocurred --- src/watcher.ts | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/watcher.ts b/src/watcher.ts index 3179967..a672d51 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -1,3 +1,4 @@ +import path from "path"; import * as fs from "fs"; import chokidar from "chokidar"; import * as matcher from "closest-match"; @@ -9,11 +10,11 @@ const outputFolder = process.env.E2G_OUTPUT_FOLDER || "/var/e2g-output"; console.log(`[i] Watching ${inputFolder}..\n`); -chokidar.watch(inputFolder, { usePolling: true }).on("add", path => { +chokidar.watch(inputFolder, { usePolling: true }).on("add", filePath => { - console.log(`[i] Found ${path}!`); + console.log(`[i] Found ${filePath}!`); - const fileContents = fs.readFileSync(path, "utf-8"); + const fileContents = fs.readFileSync(filePath, "utf-8"); const closestMatch = matcher.closestMatch(fileContents.split("\n")[0], [...headers.keys()]); @@ -28,13 +29,21 @@ chokidar.watch(inputFolder, { usePolling: true }).on("add", path => { console.log(`[i] Determined the file type to be of kind '${converter}'.`); // Determine convertor type and run conversion. - createAndRunConverter(converter, path, outputFolder, () => { + createAndRunConverter(converter, filePath, outputFolder, + () => { - // After conversion was succesful, remove input file. - console.log(`[i] Finished converting ${path}, removing file..\n\n`); - fs.rmSync(path); - }, () => { }); + // After conversion was succesful, remove input file. + console.log(`[i] Finished converting ${path}, removing file..\n\n`); + fs.rmSync(filePath); + + }, (err) => { + console.log("[e] An error ocurred while processing. Moving file to output"); + console.log(`[e] Error details: ${err}`); + + const errorFilePath = path.join(outputFolder, filePath); + fs.renameSync(filePath, errorFilePath); + }); }); // Prep header set. From 5444ddf76a4238b13aa57967db40b930bf0a5a60 Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Fri, 2 Feb 2024 10:40:41 +0100 Subject: [PATCH 28/31] Watcher now runs once by default --- src/watcher.ts | 81 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 55 insertions(+), 26 deletions(-) diff --git a/src/watcher.ts b/src/watcher.ts index a672d51..dcea1de 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -7,44 +7,73 @@ import { createAndRunConverter } from "./converter"; // Define input and output. const inputFolder = process.env.E2G_INPUT_FOLDER || "/var/e2g-input"; const outputFolder = process.env.E2G_OUTPUT_FOLDER || "/var/e2g-output"; +const usePolling = Boolean(process.env.E2G_USE_POLLING) || false; -console.log(`[i] Watching ${inputFolder}..\n`); +console.log(`[i] Watching ${inputFolder}${usePolling ? " (using polling)" : ""}..`); -chokidar.watch(inputFolder, { usePolling: true }).on("add", filePath => { +let isProcessing = false; - console.log(`[i] Found ${filePath}!`); +chokidar + .watch(inputFolder, { usePolling: usePolling }) + .on("add", filePath => { - const fileContents = fs.readFileSync(filePath, "utf-8"); + isProcessing = true; - const closestMatch = matcher.closestMatch(fileContents.split("\n")[0], [...headers.keys()]); + console.log(`[i] Found ${filePath}!`); - let converterKey = closestMatch as string; + const fileContents = fs.readFileSync(filePath, "utf-8"); - // If multiple matches were found (type would not be 'string'), pick the first. - if (typeof closestMatch !== "string") { - converterKey = closestMatch[0]; - } + const closestMatch = matcher.closestMatch(fileContents.split("\n")[0], [...headers.keys()]); - const converter = headers.get(converterKey); - console.log(`[i] Determined the file type to be of kind '${converter}'.`); + let converterKey = closestMatch as string; - // Determine convertor type and run conversion. - createAndRunConverter(converter, filePath, outputFolder, - () => { + // If multiple matches were found (type would not be 'string'), pick the first. + if (typeof closestMatch !== "string") { + converterKey = closestMatch[0]; + } - // After conversion was succesful, remove input file. - console.log(`[i] Finished converting ${path}, removing file..\n\n`); - fs.rmSync(filePath); - - }, (err) => { + const converter = headers.get(converterKey); + console.log(`[i] Determined the file type to be of kind '${converter}'.`); - console.log("[e] An error ocurred while processing. Moving file to output"); - console.log(`[e] Error details: ${err}`); + // Determine convertor type and run conversion. + createAndRunConverter(converter, filePath, outputFolder, + () => { - const errorFilePath = path.join(outputFolder, filePath); - fs.renameSync(filePath, errorFilePath); - }); -}); + // After conversion was succesful, remove input file. + console.log(`[i] Finished converting ${path}, removing file..\n\n`); + fs.rmSync(filePath); + + isProcessing = false; + + if (!usePolling) { + process.exit(0); + } + + }, (err) => { + + console.log("[e] An error ocurred while processing. Moving file to output"); + console.log(`[e] Error details: ${err}`); + + const errorFilePath = path.join(outputFolder, filePath); + fs.renameSync(filePath, errorFilePath); + + isProcessing = false; + + if (!usePolling) { + process.exit(0); + } + }); + }) + .on("ready", () => { + + // When polling was not set to true (thus runOnce) and there is no file currently being processed, stop the container. + setTimeout(() => { + if (!usePolling && !isProcessing) { + console.log("[i] Found no file to convert, stop container as usePolling is set to false.."); + process.exit(0); + } + }, 5000); + }); // Prep header set. const headers: Map = new Map(); From 1efa91f8a41bff4a9ca284fadc250937466a2a64 Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Fri, 2 Feb 2024 10:59:36 +0100 Subject: [PATCH 29/31] Remove unused dep, cleanup some console logs --- Dockerfile | 2 +- package.json | 1 - src/converter.ts | 6 ++++++ src/watcher.ts | 15 ++++++++++----- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 36cbff7..5532757 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-alpine3.19 +FROM node:20-alpine3.19 WORKDIR /app diff --git a/package.json b/package.json index 768c66e..2f2f72e 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ "chokidar": "^3.5.3", "cli-progress": "^3.12.0", "closest-match": "^1.3.3", - "cross-fetch": "^4.0.0", "csv-parse": "^5.5.2", "dayjs": "^1.11.10", "dotenv": "^16.3.1", diff --git a/src/converter.ts b/src/converter.ts index 163fb48..a20de21 100644 --- a/src/converter.ts +++ b/src/converter.ts @@ -11,6 +11,11 @@ import { FinpensionConverter } from "./converters/finpensionConverter"; export function createAndRunConverter(converterType: string, inputFilePath: string, outputFilePath: string, completionCallback: CallableFunction, errorCallback: CallableFunction) { + // Verify if Ghostolio account ID is set (because without it there can be no valid output). + if (!process.env.GHOSTFOLIO_ACCOUNT_ID) { + return errorCallback(new Error("Environment variable GHOSTFOLIO_ACCOUNT_ID not set!")); + } + const converterTypeLc = converterType.toLocaleLowerCase(); // Determine convertor type. @@ -29,6 +34,7 @@ export function createAndRunConverter(converterType: string, inputFilePath: stri console.log(`[i] Wrote data to '${outputFileName}.json'!`); completionCallback(); + }, (error) => errorCallback(error)); } diff --git a/src/watcher.ts b/src/watcher.ts index dcea1de..ddc04ae 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -19,7 +19,7 @@ chokidar isProcessing = true; - console.log(`[i] Found ${filePath}!`); + console.log(`[i] Found ${path.basename(filePath)}!`); const fileContents = fs.readFileSync(filePath, "utf-8"); @@ -40,26 +40,31 @@ chokidar () => { // After conversion was succesful, remove input file. - console.log(`[i] Finished converting ${path}, removing file..\n\n`); + console.log(`[i] Finished converting ${path.basename(filePath)}, removing file..`); fs.rmSync(filePath); isProcessing = false; if (!usePolling) { + console.log("[i] Stop container as usePolling is set to false.."); process.exit(0); } }, (err) => { - console.log("[e] An error ocurred while processing. Moving file to output"); + console.log("[e] An error ocurred while processing."); console.log(`[e] Error details: ${err}`); - const errorFilePath = path.join(outputFolder, filePath); - fs.renameSync(filePath, errorFilePath); + // Move file with errors to output folder so it can be fixed manually. + console.log("[e] Moving file to output.."); + const errorFilePath = path.join(outputFolder, path.basename(filePath)); + fs.copyFileSync(filePath, errorFilePath); + fs.rmSync(filePath); isProcessing = false; if (!usePolling) { + console.log("[i] Stop container as usePolling is set to false.."); process.exit(0); } }); From ff3ace66eb05c74e2fe6969077032f0b761c89ea Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Fri, 2 Feb 2024 11:05:25 +0100 Subject: [PATCH 30/31] Rename some files, add error handling to manual.ts --- .../workflows/{main.yml => frameworkTesting.yml} | 0 nodemon.json | 6 ------ package.json | 2 +- src/index.ts | 9 --------- src/manual.ts | 16 ++++++++++++++++ 5 files changed, 17 insertions(+), 16 deletions(-) rename .github/workflows/{main.yml => frameworkTesting.yml} (100%) delete mode 100644 nodemon.json delete mode 100644 src/index.ts create mode 100644 src/manual.ts diff --git a/.github/workflows/main.yml b/.github/workflows/frameworkTesting.yml similarity index 100% rename from .github/workflows/main.yml rename to .github/workflows/frameworkTesting.yml diff --git a/nodemon.json b/nodemon.json deleted file mode 100644 index 5131f09..0000000 --- a/nodemon.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "watch": ["src"], - "ext": ".ts,.js", - "ignore": [], - "exec": "ts-node ./src/index.ts" - } \ No newline at end of file diff --git a/package.json b/package.json index 2f2f72e..d1ef420 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "Convert multiple broker exports to Ghostfolio import", "scripts": { - "start": "ts-node ./src/index.ts", + "start": "ts-node ./src/manual.ts", "watch": "ts-node ./src/watcher.ts", "test": "jest --coverage" }, diff --git a/src/index.ts b/src/index.ts deleted file mode 100644 index b4f7c61..0000000 --- a/src/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { createAndRunConverter } from "./converter"; - -require("dotenv").config(); - -// Define import file path. -const inputFile = process.env.INPUT_FILE; - -// Determine convertor type and run conversion. -createAndRunConverter(process.argv[2].toLocaleLowerCase(), inputFile, ".", () => { }, () => { }); diff --git a/src/manual.ts b/src/manual.ts new file mode 100644 index 0000000..1896e67 --- /dev/null +++ b/src/manual.ts @@ -0,0 +1,16 @@ +import { createAndRunConverter } from "./converter"; + +// Check if converter was specified. +if (process.argv.length != 3) { + console.log("[e] Invalid run command: converter not specified!");; +} +else { + + require("dotenv").config(); + + // Define import file path. + const inputFile = process.env.INPUT_FILE; + + // Determine convertor type and run conversion. + createAndRunConverter(process.argv[2].toLocaleLowerCase(), inputFile, ".", () => { }, () => { }); +} From b33e85f385dda231d25588bf83987cda33367e81 Mon Sep 17 00:00:00 2001 From: Dick Wolff Date: Fri, 2 Feb 2024 11:24:56 +0100 Subject: [PATCH 31/31] Add reamde instructions for Docker --- README.md | 49 +++++++++++++++++++++++++++++++++++++++++++++---- src/watcher.ts | 2 +- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 983f546..c8d95e9 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,47 @@ This tool allows you to convert a multiple transaction exports (CSV) to an impor Is your broker not in the list? Feel free to create an [issue](https://github.com/dickwolff/Export-To-Ghostfolio/issues/new) or, even better, build it yourself and create a [pull request](https://github.com/dickwolff/Export-To-Ghostfolio/compare)! -## System requirements +## How to use -The tool requires you to install the latest LTS version of Node, which you can download [here](https://nodejs.org/en/download/). The tool can run on any OS on which you can install Node. +You can run the tool on your local machine by cloning this repository. You can also run the tool inside a Docker container. See the runtime specific instructions below. -## How to use +## Docker + +
+View instructions + +### System requirements + +To run the Docker container you need to have [Docker](https://docs.docker.com/get-docker/) installed on your machine. The image is publishe to [Docker Hub](https://hub.docker.com/repository/docker/dickwolff/export-to-ghostfolio/). You can then run the image like: + +``` +docker run -d -v /C/.../docker_in:/var/e2g-input -v /C/.../docker_out:/var/e2g-output --env GHOSTFOLIO_ACCOUNT_ID=xxxxxxx dickwolff/export-to-ghostfolio +``` + +The following parameters can be given to the Docker run command. + +| Command | Optional | Description | +| ------- | -------- | ----------- | +| ` -v {local_in-folder}:/var/e2g-input` | N | The input folder where you put the files to be processed | +| `-v {local_out_folder}:/var/e2g-output` | N | The output folder where the Ghostfolio import JSON will be placed. Also the input file will be moved here when an error ocurred while processing the file. | +| `--env GHOSTFOLIO_ACCOUNT_ID=xxxxxxx` | N | Your Ghostolio account ID 1 | +| `--env USE_POLLING=true` | Y | When set to true, the container will continously look for new files to process and the container will not stop. | +| `--env DEBUG_LOGGING=true` | Y | When set to true, the container will show logs in more detail, useful for error tracing. | + +1: You can retrieve your Ghostfolio account ID by going to Accounts > select your account and copying the ID from the URL. + +![image](https://user-images.githubusercontent.com/5620002/203353840-f5db7323-fb2f-4f4f-befc-e4e340466a74.png) + +
+ +## Run locally + +
+View instructions + +### System requirements + +The tool requires you to install the latest LTS version of Node, which you can download [here](https://nodejs.org/en/download/). The tool can run on any OS on which you can install Node. ### Download transaction export @@ -41,6 +77,7 @@ Login to your Finpension account. Select your portfolio from the landing page. T Login to your Swissquote account. From the bar menu click on “Transactions”. Select the desired time period as well as types and then select the “export CSV” button to the right. #### Schwab + Login to your Schwab account. Go to “Accounts” then “History”. Select the account you want to download details from. Select the “Date Range” and select “Export” (csv). Save the file. ![Export instructions for Schwab](./assets/export-schwab.jpg) @@ -60,7 +97,7 @@ The repository contains a sample `.env` file. Rename this from `.env.sample`. ![image](https://user-images.githubusercontent.com/5620002/203353840-f5db7323-fb2f-4f4f-befc-e4e340466a74.png) - Optionally you can enable debug logging by setting the `DEBUG_LOGGING` variable to `TRUE`. -You can now run `npm run start [exporttype]`. See the table with run commands below. The tool will open your export and will convert this. It retrieves the symbols that are supported with YAHOO Finance (e.g. for European stocks like `ASML`, it will retrieve `ASML.AS` by the corresponding ISIN). +You can now run `npm run start [exporttype]`. See the table with run commands below. The tool will open your export and will convert this. It retrieves the symbols that are supported with YAHOO Finance (e.g. for European stocks like `ASML`, it will retrieve `ASML.AS` by the corresponding ISIN). | Exporter | Run command | | ----------- | ----------------------------------- | @@ -70,6 +107,10 @@ You can now run `npm run start [exporttype]`. See the table with run commands be | Swissquote | `run start swissquote` (or `sq`) | | Schwab | `run start schwab` | +
+ +## Import to Ghostfolio + The export file can now be imported in Ghostfolio by going to Portfolio > Activities and pressing the 3 dots at the top right of the table. Since Ghostfolio 1.221.0, you can now preview the import and validate the data has been converted correctly. When it is to your satisfaction, press import to add the activities to your portfolio. ![image](https://user-images.githubusercontent.com/5620002/203356387-1f42ca31-7cff-44a5-8f6c-84045cf7101e.png) diff --git a/src/watcher.ts b/src/watcher.ts index ddc04ae..91ed1c9 100644 --- a/src/watcher.ts +++ b/src/watcher.ts @@ -7,7 +7,7 @@ import { createAndRunConverter } from "./converter"; // Define input and output. const inputFolder = process.env.E2G_INPUT_FOLDER || "/var/e2g-input"; const outputFolder = process.env.E2G_OUTPUT_FOLDER || "/var/e2g-output"; -const usePolling = Boolean(process.env.E2G_USE_POLLING) || false; +const usePolling = Boolean(process.env.USE_POLLING) || false; console.log(`[i] Watching ${inputFolder}${usePolling ? " (using polling)" : ""}..`);